incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [3/3] git commit: BLUR-ID:36 fix compilation errors
Date Thu, 18 Oct 2012 20:10:13 GMT
BLUR-ID:36 fix compilation errors

Signed-off-by: Aaron McCurry <amccurry@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/b3cc2d3a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/b3cc2d3a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/b3cc2d3a

Branch: refs/heads/lucene-4.0.0
Commit: b3cc2d3abd593301e15df20096c6d7298ffdfdc9
Parents: f406fb5
Author: Gagan <gagandeepjuneja@gmail.com>
Authored: Thu Oct 18 11:56:29 2012 +0530
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Thu Oct 18 15:40:06 2012 -0400

----------------------------------------------------------------------
 .../java/org/apache/blur/manager/IndexManager.java |  109 +++++++--------
 .../org/apache/blur/utils/TermDocIterable.java     |   35 +++---
 .../org/apache/blur/utils/TermDocIterableTest.java |   46 ++++---
 3 files changed, 96 insertions(+), 94 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b3cc2d3a/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java b/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
index 725174a..3bd33cf 100644
--- a/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
+++ b/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
@@ -86,16 +86,18 @@ import org.apache.blur.utils.ForkJoin.ParallelCall;
 import org.apache.blur.utils.TermDocIterable;
 import org.apache.hadoop.io.BytesWritable;
 import org.apache.lucene.document.Document;
-import org.apache.lucene.document.FieldSelector;
-import org.apache.lucene.document.FieldSelectorResult;
+import org.apache.lucene.index.AtomicReader;
 import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.StoredFieldVisitor;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermDocs;
-import org.apache.lucene.index.TermEnum;
-import org.apache.lucene.queryParser.ParseException;
+import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.queryparser.classic.ParseException;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.Filter;
@@ -104,7 +106,8 @@ import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.util.ReaderUtil;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
 
 public class IndexManager {
 
@@ -121,7 +124,7 @@ public class IndexManager {
   private BlurFilterCache _filterCache = new DefaultBlurFilterCache();
   private BlurMetrics _blurMetrics;
   private QueryMetrics _queryMetrics;
-  private long _defaultParallelCallTimeout = TimeUnit.MINUTES.toMillis(1);
+  private long _defaultParallelCallTimeout = TimeUnit.MINUTES.toMillis(1):
 
   public void setMaxClauseCount(int maxClauseCount) {
     BooleanQuery.setMaxClauseCount(maxClauseCount);
@@ -409,21 +412,23 @@ public class IndexManager {
     if (docId >= reader.maxDoc()) {
       throw new RuntimeException("Location id [" + locationId + "] with docId [" + docId
+ "] is not valid.");
     }
+    Bits liveDocs = MultiFields.getLiveDocs(reader);
     if (selector.isRecordOnly()) {
       // select only the row for the given data or location id.
-      if (reader.isDeleted(docId)) {
+      if (!liveDocs.get(docId)) {
         fetchResult.exists = false;
         fetchResult.deleted = true;
         return;
       } else {
         fetchResult.exists = true;
         fetchResult.deleted = false;
-        Document document = reader.document(docId, getFieldSelector(selector));
+        reader.document(docId, getFieldSelector(selector));
+        Document document = reader.document(docId);
         fetchResult.recordResult = getColumns(document);
         return;
       }
     } else {
-      if (reader.isDeleted(docId)) {
+      if (!liveDocs.get(docId)) {
         fetchResult.exists = false;
         fetchResult.deleted = true;
         return;
@@ -431,26 +436,27 @@ public class IndexManager {
         fetchResult.exists = true;
         fetchResult.deleted = false;
         String rowId = getRowId(reader, docId);
-        TermDocs termDocs = reader.termDocs(new Term(ROW_ID, rowId));
+        DocsEnum termDocs = ((AtomicReader)reader).termDocsEnum(new Term(ROW_ID, rowId));
         fetchResult.rowResult = new FetchRowResult(getRow(new TermDocIterable(termDocs, reader,
getFieldSelector(selector))));
         return;
       }
     }
   }
 
-  private static String getRowId(IndexReader reader, int docId) throws CorruptIndexException,
IOException {
-    Document document = reader.document(docId, new FieldSelector() {
+  private static String getRowId(IndexReader reader, int docId)
+      throws CorruptIndexException, IOException {
+    reader.document(docId, new StoredFieldVisitor() {
       private static final long serialVersionUID = 4912420100148752051L;
 
       @Override
-      public FieldSelectorResult accept(String fieldName) {
-        if (ROW_ID.equals(fieldName)) {
-          return FieldSelectorResult.LOAD_AND_BREAK;
+      public Status needsField(FieldInfo fieldInfo) throws IOException {
+        if (ROW_ID.equals(fieldInfo.name)) {
+          return StoredFieldVisitor.Status.STOP;
         }
-        return FieldSelectorResult.NO_LOAD;
+        return StoredFieldVisitor.Status.NO;
       }
     });
-    return document.get(ROW_ID);
+    return reader.document(docId).get(ROW_ID);
   }
 
   private static String getColumnName(String fieldName) {
@@ -461,42 +467,41 @@ public class IndexManager {
     return fieldName.substring(0, fieldName.lastIndexOf('.'));
   }
 
-  public static FieldSelector getFieldSelector(final Selector selector) {
-    return new FieldSelector() {
+  public static StoredFieldVisitor getFieldSelector(final Selector selector) {
+    return new StoredFieldVisitor() {
       private static final long serialVersionUID = 4089164344758433000L;
 
       @Override
-      public FieldSelectorResult accept(String fieldName) {
-        if (ROW_ID.equals(fieldName)) {
-          return FieldSelectorResult.LOAD;
+      public Status needsField(FieldInfo fieldInfo) throws IOException {
+        if (ROW_ID.equals(fieldInfo.name)) {
+          return StoredFieldVisitor.Status.YES;
         }
-        if (RECORD_ID.equals(fieldName)) {
-          return FieldSelectorResult.LOAD;
+        if (RECORD_ID.equals(fieldInfo.name)) {
+          return StoredFieldVisitor.Status.YES;
         }
-        if (PRIME_DOC.equals(fieldName)) {
-          return FieldSelectorResult.NO_LOAD;
+        if (PRIME_DOC.equals(fieldInfo.name)) {
+          return StoredFieldVisitor.Status.NO;
         }
         if (selector.columnFamiliesToFetch == null && selector.columnsToFetch ==
null) {
-          return FieldSelectorResult.LOAD;
+          return StoredFieldVisitor.Status.YES;
         }
-        String columnFamily = getColumnFamily(fieldName);
+        String columnFamily = getColumnFamily(fieldInfo.name);
         if (selector.columnFamiliesToFetch != null) {
           if (selector.columnFamiliesToFetch.contains(columnFamily)) {
-            return FieldSelectorResult.LOAD;
+            return StoredFieldVisitor.Status.YES;
           }
         }
-        String columnName = getColumnName(fieldName);
+        String columnName = getColumnName(fieldInfo.name);
         if (selector.columnsToFetch != null) {
           Set<String> columns = selector.columnsToFetch.get(columnFamily);
           if (columns != null && columns.contains(columnName)) {
-            return FieldSelectorResult.LOAD;
+            return StoredFieldVisitor.Status.YES;
           }
         }
-        return FieldSelectorResult.NO_LOAD;
-      }
-    };
+        return StoredFieldVisitor.Status.NO;
+    }
+  };
   }
-
   public IndexServer getIndexServer() {
     return _indexServer;
   }
@@ -575,29 +580,21 @@ public class IndexManager {
     return reader.docFreq(getTerm(columnFamily, columnName, value));
   }
 
-  public static List<String> terms(IndexReader reader, String columnFamily, String
columnName, String startWith, short size) throws IOException {
+  public static List<String> terms(IndexReader reader, String columnFamily,
+      String columnName, String startWith, short size) throws IOException {
     Term term = getTerm(columnFamily, columnName, startWith);
     String field = term.field();
     List<String> terms = new ArrayList<String>(size);
-    TermEnum termEnum = reader.terms(term);
-    try {
-      do {
-        Term currentTerm = termEnum.term();
-        if (currentTerm == null) {
-          return terms;
-        }
-        if (!currentTerm.field().equals(field)) {
-          break;
-        }
-        terms.add(currentTerm.text());
-        if (terms.size() >= size) {
-          return terms;
-        }
-      } while (termEnum.next());
-      return terms;
-    } finally {
-      termEnum.close();
+    Terms termsAll = ((AtomicReader) reader).terms(term.field());
+    TermsEnum termEnum = termsAll.iterator(null);
+    BytesRef currentTermText;
+    while ((currentTermText = termEnum.next()) != null) {
+      terms.add(currentTermText.toString());
+      if (terms.size() >= size) {
+        return terms;
+      }
     }
+    return terms;
   }
 
   private static Term getTerm(String columnFamily, String columnName, String value) {
@@ -617,7 +614,7 @@ public class IndexManager {
     for (BlurIndex blurIndex : blurIndexes.values()) {
       IndexReader reader = blurIndex.getIndexReader();
       try {
-        FieldInfos mergedFieldInfos = ReaderUtil.getMergedFieldInfos(reader);
+        FieldInfos mergedFieldInfos = MultiFields.getMergedFieldInfos(reader);
         for (FieldInfo fieldInfo : mergedFieldInfos) {
           String fieldName = fieldInfo.name;
           int index = fieldName.indexOf('.');

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b3cc2d3a/src/blur-core/src/main/java/org/apache/blur/utils/TermDocIterable.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/utils/TermDocIterable.java b/src/blur-core/src/main/java/org/apache/blur/utils/TermDocIterable.java
index 8312360..9f9efff 100644
--- a/src/blur-core/src/main/java/org/apache/blur/utils/TermDocIterable.java
+++ b/src/blur-core/src/main/java/org/apache/blur/utils/TermDocIterable.java
@@ -20,20 +20,20 @@ import java.io.IOException;
 import java.util.Iterator;
 
 import org.apache.lucene.document.Document;
-import org.apache.lucene.document.FieldSelector;
+import org.apache.lucene.index.DocsEnum;
 import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.TermDocs;
+import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.util.Bits;
 
 public class TermDocIterable implements Iterable<Document> {
 
-  private TermDocs termDocs;
+  private DocsEnum docsEnum;
   private IndexReader reader;
-  private FieldSelector fieldSelector;
 
-  public TermDocIterable(TermDocs termDocs, IndexReader reader, FieldSelector fieldSelector)
{
-    this.termDocs = termDocs;
+  public TermDocIterable(DocsEnum docsEnum, IndexReader reader) {
+    this.docsEnum = docsEnum;
     this.reader = reader;
-    this.fieldSelector = fieldSelector;
   }
 
   @Override
@@ -66,23 +66,22 @@ public class TermDocIterable implements Iterable<Document> {
   }
 
   private Document getDoc() throws IOException {
-    return reader.document(termDocs.doc(), fieldSelector);
+    return reader.document(docsEnum.docID());
   }
 
   private boolean getNext() {
     try {
-      boolean next = termDocs.next();
-      if (!next) {
-        termDocs.close();
-        return next;
+      int next = docsEnum.nextDoc();
+      if (next == DocIdSetIterator.NO_MORE_DOCS) {
+        return false;
       }
-      while (reader.isDeleted(termDocs.doc())) {
-        next = termDocs.next();
-      }
-      if (!next) {
-        termDocs.close();
+      Bits liveDocs = MultiFields.getLiveDocs(reader);
+      if(liveDocs != null){
+        while (!liveDocs.get(docsEnum.docID())) {
+          next = docsEnum.nextDoc();
+        }
       }
-      return next;
+      return next == DocIdSetIterator.NO_MORE_DOCS ? false : true;
     } catch (IOException e) {
       throw new RuntimeException(e);
     }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b3cc2d3a/src/blur-core/src/test/java/org/apache/blur/utils/TermDocIterableTest.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/org/apache/blur/utils/TermDocIterableTest.java b/src/blur-core/src/test/java/org/apache/blur/utils/TermDocIterableTest.java
index 610fccc..58e4ed8 100644
--- a/src/blur-core/src/test/java/org/apache/blur/utils/TermDocIterableTest.java
+++ b/src/blur-core/src/test/java/org/apache/blur/utils/TermDocIterableTest.java
@@ -17,25 +17,24 @@ package org.apache.blur.utils;
  * limitations under the License.
  */
 
-import static org.apache.blur.lucene.LuceneConstant.LUCENE_VERSION;
+import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION;
 import static org.junit.Assert.assertEquals;
 
 import java.io.File;
 import java.io.IOException;
 
-import org.apache.blur.utils.TermDocIterable;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldSelector;
-import org.apache.lucene.document.FieldSelectorResult;
-import org.apache.lucene.document.Field.Index;
-import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.index.AtomicReader;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.DocsEnum;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.SlowCompositeReaderWrapper;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermDocs;
 import org.apache.lucene.store.FSDirectory;
 import org.junit.Before;
 import org.junit.Test;
@@ -52,18 +51,14 @@ public class TermDocIterableTest {
     reader = createIndexReader();
   }
 
-  @SuppressWarnings("serial")
   @Test
   public void testTermDocIterable() throws IOException {
+    AtomicReader atomicReader = (AtomicReader)reader;
     for (int pass = 0; pass < 1; pass++) {
       for (int id = 0; id < BLOCKS; id++) {
-        TermDocs termDocs = reader.termDocs(new Term("id", Integer.toString(id)));
-        TermDocIterable iterable = new TermDocIterable(termDocs, reader, new FieldSelector()
{
-          @Override
-          public FieldSelectorResult accept(String fieldName) {
-            return FieldSelectorResult.LOAD;
-          }
-        });
+        
+        DocsEnum termDocs = atomicReader.termDocsEnum(new Term("id", Integer.toString(id)));
+        TermDocIterable iterable = new TermDocIterable(termDocs, atomicReader);
         int count = 0;
         int i = 0;
         long s = System.nanoTime();
@@ -81,7 +76,7 @@ public class TermDocIterableTest {
 
   private IndexReader createIndexReader() throws IOException {
     FSDirectory directory = FSDirectory.open(new File("./tmp/termdociterable"));
-    if (!IndexReader.indexExists(directory)) {
+    if (!DirectoryReader.indexExists(directory)) {
       rm(new File("./tmp/termdociterable"));
       IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(LUCENE_VERSION,
new StandardAnalyzer(LUCENE_VERSION)));
       for (int i = 0; i < BLOCKS; i++) {
@@ -89,7 +84,10 @@ public class TermDocIterableTest {
       }
       writer.close();
     }
-    return IndexReader.open(directory);
+    IndexReader indexReader = DirectoryReader.open(directory);
+    int numDocs = indexReader.numDocs();
+    System.out.println("numDocs " +numDocs);
+    return SlowCompositeReaderWrapper.wrap(DirectoryReader.open(directory));
   }
 
   private File rm(File file) {
@@ -103,12 +101,20 @@ public class TermDocIterableTest {
   }
 
   private void addDocumentBlock(int id, int count, IndexWriter writer) throws IOException
{
+    FieldType fieldType = new FieldType();
+    fieldType.setStored(true);   
+    
+    FieldType fieldTypeNoIndex = new FieldType();
+    fieldTypeNoIndex.setStored(true);
+    fieldTypeNoIndex.setIndexed(false);
+    
+    
     for (int i = 0; i < count; i++) {
       Document document = new Document();
-      document.add(new Field("id", Integer.toString(id), Store.YES, Index.NOT_ANALYZED_NO_NORMS));
-      document.add(new Field("field", Integer.toString(i), Store.YES, Index.NOT_ANALYZED_NO_NORMS));
+      document.add(new Field("id", Integer.toString(id), fieldType));
+      document.add(new Field("field", Integer.toString(i), fieldType));
       for (int j = 0; j < 100; j++) {
-        document.add(new Field("field" + j, "testing here testing here testing here testing
here testing here testing here testing here", Store.YES, Index.NO));
+        document.add(new Field("field" + j, "testing here testing here testing here testing
here testing here testing here testing here", fieldTypeNoIndex));
       }
       writer.addDocument(document);
     }


Mime
View raw message