incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [2/3] git commit: All classes now compile in blur-core.
Date Thu, 18 Oct 2012 20:10:13 GMT
All classes now compile in blur-core.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/d580e345
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/d580e345
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/d580e345

Branch: refs/heads/lucene-4.0.0
Commit: d580e345c82b62361af90966aa3813ba23266669
Parents: b3cc2d3
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Thu Oct 18 16:08:54 2012 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Thu Oct 18 16:08:54 2012 -0400

----------------------------------------------------------------------
 .../java/org/apache/blur/manager/IndexManager.java |   39 ++--
 .../manager/writer/lucene/SnapshotIndexReader.java |  170 ------------
 .../writer/lucene/SoftDeleteIndexReader.java       |  213 ---------------
 .../org/apache/blur/utils/TermDocIterable.java     |   12 +-
 4 files changed, 27 insertions(+), 407 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d580e345/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java b/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
index 3bd33cf..c711d7b 100644
--- a/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
+++ b/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
@@ -20,8 +20,6 @@ import static org.apache.blur.utils.BlurConstants.PRIME_DOC;
 import static org.apache.blur.utils.BlurConstants.RECORD_ID;
 import static org.apache.blur.utils.BlurConstants.ROW_ID;
 import static org.apache.blur.utils.BlurUtil.findRecordMutation;
-import static org.apache.blur.utils.BlurUtil.readFilter;
-import static org.apache.blur.utils.BlurUtil.readQuery;
 import static org.apache.blur.utils.RowDocumentUtil.getColumns;
 import static org.apache.blur.utils.RowDocumentUtil.getRow;
 
@@ -48,7 +46,6 @@ import org.apache.blur.analysis.BlurAnalyzer;
 import org.apache.blur.concurrent.Executors;
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
-import org.apache.blur.lucene.EscapeRewrite;
 import org.apache.blur.lucene.search.FacetQuery;
 import org.apache.blur.manager.results.BlurResultIterable;
 import org.apache.blur.manager.results.BlurResultIterableSearcher;
@@ -86,15 +83,16 @@ import org.apache.blur.utils.ForkJoin.ParallelCall;
 import org.apache.blur.utils.TermDocIterable;
 import org.apache.hadoop.io.BytesWritable;
 import org.apache.lucene.document.Document;
+import org.apache.lucene.document.DocumentStoredFieldVisitor;
 import org.apache.lucene.index.AtomicReader;
 import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.DocsEnum;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.MultiFields;
 import org.apache.lucene.index.StoredFieldVisitor;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.index.DocsEnum;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.queryparser.classic.ParseException;
@@ -124,7 +122,7 @@ public class IndexManager {
   private BlurFilterCache _filterCache = new DefaultBlurFilterCache();
   private BlurMetrics _blurMetrics;
   private QueryMetrics _queryMetrics;
-  private long _defaultParallelCallTimeout = TimeUnit.MINUTES.toMillis(1):
+  private long _defaultParallelCallTimeout = TimeUnit.MINUTES.toMillis(1);
 
   public void setMaxClauseCount(int maxClauseCount) {
     BooleanQuery.setMaxClauseCount(maxClauseCount);
@@ -351,11 +349,11 @@ public class IndexManager {
   }
 
   private Filter getFilter(ExpertQuery expertQuery) throws BException {
-    return readFilter(expertQuery.getFilter());
+    throw new BException("Expert query not supported", new Throwable());
   }
 
   private Query getQuery(ExpertQuery expertQuery) throws BException {
-    return readQuery(expertQuery.getQuery());
+    throw new BException("Expert query not supported", new Throwable());
   }
 
   private boolean isSimpleQuery(BlurQuery blurQuery) {
@@ -436,18 +434,15 @@ public class IndexManager {
         fetchResult.exists = true;
         fetchResult.deleted = false;
         String rowId = getRowId(reader, docId);
-        DocsEnum termDocs = ((AtomicReader)reader).termDocsEnum(new Term(ROW_ID, rowId));
+        DocsEnum termDocs = ((AtomicReader) reader).termDocsEnum(new Term(ROW_ID, rowId));
         fetchResult.rowResult = new FetchRowResult(getRow(new TermDocIterable(termDocs, reader,
getFieldSelector(selector))));
         return;
       }
     }
   }
 
-  private static String getRowId(IndexReader reader, int docId)
-      throws CorruptIndexException, IOException {
+  private static String getRowId(IndexReader reader, int docId) throws CorruptIndexException,
IOException {
     reader.document(docId, new StoredFieldVisitor() {
-      private static final long serialVersionUID = 4912420100148752051L;
-
       @Override
       public Status needsField(FieldInfo fieldInfo) throws IOException {
         if (ROW_ID.equals(fieldInfo.name)) {
@@ -467,10 +462,8 @@ public class IndexManager {
     return fieldName.substring(0, fieldName.lastIndexOf('.'));
   }
 
-  public static StoredFieldVisitor getFieldSelector(final Selector selector) {
-    return new StoredFieldVisitor() {
-      private static final long serialVersionUID = 4089164344758433000L;
-
+  public static DocumentStoredFieldVisitor getFieldSelector(final Selector selector) {
+    return new DocumentStoredFieldVisitor() {
       @Override
       public Status needsField(FieldInfo fieldInfo) throws IOException {
         if (ROW_ID.equals(fieldInfo.name)) {
@@ -499,9 +492,10 @@ public class IndexManager {
           }
         }
         return StoredFieldVisitor.Status.NO;
-    }
-  };
+      }
+    };
   }
+
   public IndexServer getIndexServer() {
     return _indexServer;
   }
@@ -580,8 +574,7 @@ public class IndexManager {
     return reader.docFreq(getTerm(columnFamily, columnName, value));
   }
 
-  public static List<String> terms(IndexReader reader, String columnFamily,
-      String columnName, String startWith, short size) throws IOException {
+  public static List<String> terms(IndexReader reader, String columnFamily, String
columnName, String startWith, short size) throws IOException {
     Term term = getTerm(columnFamily, columnName, startWith);
     String field = term.field();
     List<String> terms = new ArrayList<String>(size);
@@ -946,8 +939,10 @@ public class IndexManager {
         BlurIndex index = entry.getValue();
         IndexReader reader = index.getIndexReader();
         String shard = entry.getKey();
-        IndexReader escapeReader = EscapeRewrite.wrap(reader, _running);
-        IndexSearcher searcher = new IndexSearcher(escapeReader);
+        // @TODO need to add escapable rewriter
+        // IndexReader escapeReader = EscapeRewrite.wrap(reader, _running);
+        // IndexSearcher searcher = new IndexSearcher(escapeReader);
+        IndexSearcher searcher = new IndexSearcher(reader);
         searcher.setSimilarity(_indexServer.getSimilarity(_table));
         Query rewrite = searcher.rewrite((Query) _query.clone());
         return new BlurResultIterableSearcher(_running, rewrite, _table, shard, searcher,
_selector, reader);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d580e345/src/blur-core/src/main/java/org/apache/blur/manager/writer/lucene/SnapshotIndexReader.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/writer/lucene/SnapshotIndexReader.java
b/src/blur-core/src/main/java/org/apache/blur/manager/writer/lucene/SnapshotIndexReader.java
deleted file mode 100644
index 171b3b6..0000000
--- a/src/blur-core/src/main/java/org/apache/blur/manager/writer/lucene/SnapshotIndexReader.java
+++ /dev/null
@@ -1,170 +0,0 @@
-package org.apache.blur.manager.writer.lucene;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.io.IOException;
-
-import org.apache.lucene.index.FilterIndexReader;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.MultiReader;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermDocs;
-import org.apache.lucene.index.TermPositions;
-
-public class SnapshotIndexReader extends FilterIndexReader {
-
-  private final int maxDocs;
-  private final int numDocs;
-
-  public SnapshotIndexReader(IndexReader in, int maxDocs) {
-    super(in);
-    this.numDocs = in.numDocs();
-    this.maxDocs = maxDocs;
-  }
-
-  public static IndexReader wrap(IndexReader reader, int maxDocs) throws IOException {
-    IndexReader[] readers = reader.getSequentialSubReaders();
-    if (readers == null) {
-      return wrapInternal(reader, maxDocs);
-    } else {
-      IndexReader[] result = new IndexReader[readers.length];
-      for (int i = 0; i < readers.length; i++) {
-        result[i] = wrapInternal(readers[i], maxDocs);
-      }
-      return new MultiReader(result, false);
-    }
-  }
-
-  private static IndexReader wrapInternal(IndexReader reader, int maxDocs) throws IOException
{
-    return new SnapshotIndexReader(reader, maxDocs);
-  }
-
-  @Override
-  public IndexReader[] getSequentialSubReaders() {
-    return null;
-  }
-
-  @Override
-  public int numDocs() {
-    return numDocs;
-  }
-
-  @Override
-  public int maxDoc() {
-    return maxDocs;
-  }
-
-  @Override
-  public TermDocs termDocs() throws IOException {
-    return new SnapshotTermDocs(in.termDocs(), maxDocs);
-  }
-
-  public TermDocs termDocs(Term term) throws IOException {
-    ensureOpen();
-    TermDocs termDocs = termDocs();
-    termDocs.seek(term);
-    return termDocs;
-  }
-
-  @Override
-  public TermPositions termPositions() throws IOException {
-    return new SnapshotTermPositions(in.termPositions(), maxDocs);
-  }
-
-  // public TermPositions termPositions(Term term) throws IOException {
-  // ensureOpen();
-  // TermPositions termPositions = termPositions();
-  // termPositions.seek(term);
-  // return termPositions;
-  // }
-
-  public static class SnapshotTermPositions extends FilterTermPositions {
-
-    private final int maxDocs;
-
-    public SnapshotTermPositions(TermPositions termPositions, int maxDocs) {
-      super(termPositions);
-      this.maxDocs = maxDocs;
-    }
-
-    @Override
-    public boolean next() throws IOException {
-      boolean next = super.next();
-      if (next) {
-        if (doc() >= maxDocs) {
-          return false;
-        }
-      }
-      return next;
-    }
-
-    @Override
-    public int read(int[] docs, int[] freqs) throws IOException {
-      int read = super.read(docs, freqs);
-      if (read == 0) {
-        return 0;
-      }
-      if (doc() >= maxDocs) {
-        return checkResults(docs, maxDocs);
-      }
-      return read;
-    }
-  }
-
-  public static class SnapshotTermDocs extends FilterTermDocs {
-
-    private final int maxDocs;
-
-    public SnapshotTermDocs(TermDocs termDocs, int maxDocs) {
-      super(termDocs);
-      this.maxDocs = maxDocs;
-    }
-
-    @Override
-    public boolean next() throws IOException {
-      boolean next = super.next();
-      if (next) {
-        if (doc() >= maxDocs) {
-          return false;
-        }
-      }
-      return next;
-    }
-
-    @Override
-    public int read(int[] docs, int[] freqs) throws IOException {
-      int read = super.read(docs, freqs);
-      if (read == 0) {
-        return 0;
-      }
-      if (doc() >= maxDocs) {
-        return checkResults(docs, maxDocs);
-      }
-      return read;
-    }
-  }
-
-  private static int checkResults(int[] docs, int maxDocs) {
-    int length = docs.length;
-    for (int i = 0; i < length; i++) {
-      if (docs[i] >= maxDocs) {
-        return i;
-      }
-    }
-    return length;
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d580e345/src/blur-core/src/main/java/org/apache/blur/manager/writer/lucene/SoftDeleteIndexReader.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/writer/lucene/SoftDeleteIndexReader.java
b/src/blur-core/src/main/java/org/apache/blur/manager/writer/lucene/SoftDeleteIndexReader.java
deleted file mode 100644
index f81e619..0000000
--- a/src/blur-core/src/main/java/org/apache/blur/manager/writer/lucene/SoftDeleteIndexReader.java
+++ /dev/null
@@ -1,213 +0,0 @@
-package org.apache.blur.manager.writer.lucene;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.io.IOException;
-import java.util.Collection;
-
-import org.apache.lucene.index.FilterIndexReader;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.MultiReader;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermDocs;
-import org.apache.lucene.index.TermPositions;
-import org.apache.lucene.util.BitVector;
-
-public class SoftDeleteIndexReader extends FilterIndexReader {
-
-  private final boolean baseHasDeletions;
-  private final BitVector deletes;
-  private final int deleteCount;
-  private final int numDocs;
-
-  public SoftDeleteIndexReader(IndexReader in, BitVector deletes) {
-    super(in);
-    this.baseHasDeletions = in.hasDeletions();
-    if (deletes == null) {
-      throw new RuntimeException("No deletes, use regular indexreader.");
-    }
-    this.deletes = deletes;
-    this.deleteCount = deletes.count();
-    this.numDocs = in.numDocs() - deleteCount;
-  }
-
-  public static IndexReader wrap(IndexReader reader, Collection<Term> deleteTerms)
throws IOException {
-    IndexReader[] readers = reader.getSequentialSubReaders();
-    if (readers == null) {
-      return wrapInternal(reader, deleteTerms);
-    } else {
-      IndexReader[] result = new IndexReader[readers.length];
-      for (int i = 0; i < readers.length; i++) {
-        result[i] = wrapInternal(readers[i], deleteTerms);
-      }
-      return new MultiReader(result, false);
-    }
-  }
-
-  private static IndexReader wrapInternal(IndexReader reader, Collection<Term> deleteTerms)
throws IOException {
-    BitVector deletes = getDeletes(reader, deleteTerms);
-    if (deletes == null) {
-      return reader;
-    }
-    return new SoftDeleteIndexReader(reader, deletes);
-  }
-
-  private static BitVector getDeletes(IndexReader reader, Collection<Term> deleteTerms)
throws IOException {
-    BitVector deletes = null;
-    TermDocs termDocs = reader.termDocs();
-    for (Term t : deleteTerms) {
-      termDocs.seek(t);
-      while (termDocs.next()) {
-        if (deletes == null) {
-          deletes = new BitVector(reader.maxDoc());
-        }
-        int doc = termDocs.doc();
-        deletes.set(doc);
-      }
-    }
-    termDocs.close();
-    return deletes;
-  }
-
-  @Override
-  public IndexReader[] getSequentialSubReaders() {
-    return null;
-  }
-
-  @Override
-  public int numDocs() {
-    return numDocs;
-  }
-
-  @Override
-  public boolean isDeleted(int n) {
-    if (baseHasDeletions && in.isDeleted(n)) {
-      return true;
-    }
-    return deletes.get(n);
-  }
-
-  @Override
-  public boolean hasDeletions() {
-    return baseHasDeletions;
-  }
-
-  @Override
-  public TermDocs termDocs() throws IOException {
-    return new SoftDeleteTermDocs(in.termDocs(), deletes);
-  }
-
-  public TermDocs termDocs(Term term) throws IOException {
-    ensureOpen();
-    TermDocs termDocs = termDocs();
-    termDocs.seek(term);
-    return termDocs;
-  }
-
-  @Override
-  public TermPositions termPositions() throws IOException {
-    return new SoftDeleteTermPositions(in.termPositions(), deletes);
-  }
-
-  // public TermPositions termPositions(Term term) throws IOException {
-  // ensureOpen();
-  // TermPositions termPositions = termPositions();
-  // termPositions.seek(term);
-  // return termPositions;
-  // }
-
-  public static class SoftDeleteTermPositions extends FilterTermPositions {
-
-    private BitVector deletes;
-
-    public SoftDeleteTermPositions(TermPositions termPositions, BitVector deletes) {
-      super(termPositions);
-      this.deletes = deletes;
-    }
-
-    @Override
-    public boolean next() throws IOException {
-      while (super.next()) {
-        if (!deletes.get(doc())) {
-          return true;
-        }
-      }
-      return false;
-    }
-
-    @Override
-    public int read(int[] docs, int[] freqs) throws IOException {
-      int read = super.read(docs, freqs);
-      if (read == 0) {
-        return 0;
-      }
-      int validResults = removeDeletes(docs, freqs, read, deletes);
-      if (validResults == 0) {
-        return read(docs, freqs);
-      }
-      return validResults;
-    }
-  }
-
-  public static class SoftDeleteTermDocs extends FilterTermDocs {
-
-    private BitVector deletes;
-
-    public SoftDeleteTermDocs(TermDocs termDocs, BitVector deletes) {
-      super(termDocs);
-      this.deletes = deletes;
-    }
-
-    @Override
-    public boolean next() throws IOException {
-      while (super.next()) {
-        if (!deletes.get(doc())) {
-          return true;
-        }
-      }
-      return false;
-    }
-
-    @Override
-    public int read(int[] docs, int[] freqs) throws IOException {
-      int read = super.read(docs, freqs);
-      if (read == 0) {
-        return 0;
-      }
-      int validResults = removeDeletes(docs, freqs, read, deletes);
-      if (validResults == 0) {
-        return read(docs, freqs);
-      }
-      return validResults;
-    }
-  }
-
-  private static int removeDeletes(int[] docs, int[] freqs, int validLength, BitVector deletes)
{
-    int targetPosition = 0;
-    for (int i = 0; i < validLength; i++) {
-      int doc = docs[i];
-      if (!deletes.get(doc)) {
-        if (targetPosition != i) {
-          docs[targetPosition] = doc;
-          freqs[targetPosition] = freqs[i];
-        }
-        targetPosition++;
-      }
-    }
-    return targetPosition;
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d580e345/src/blur-core/src/main/java/org/apache/blur/utils/TermDocIterable.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/utils/TermDocIterable.java b/src/blur-core/src/main/java/org/apache/blur/utils/TermDocIterable.java
index 9f9efff..13b164d 100644
--- a/src/blur-core/src/main/java/org/apache/blur/utils/TermDocIterable.java
+++ b/src/blur-core/src/main/java/org/apache/blur/utils/TermDocIterable.java
@@ -20,6 +20,7 @@ import java.io.IOException;
 import java.util.Iterator;
 
 import org.apache.lucene.document.Document;
+import org.apache.lucene.document.DocumentStoredFieldVisitor;
 import org.apache.lucene.index.DocsEnum;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.MultiFields;
@@ -30,10 +31,16 @@ public class TermDocIterable implements Iterable<Document> {
 
   private DocsEnum docsEnum;
   private IndexReader reader;
+  private DocumentStoredFieldVisitor fieldSelector;
 
   public TermDocIterable(DocsEnum docsEnum, IndexReader reader) {
+    this(docsEnum, reader, new DocumentStoredFieldVisitor());
+  }
+
+  public TermDocIterable(DocsEnum docsEnum, IndexReader reader, DocumentStoredFieldVisitor
fieldSelector) {
     this.docsEnum = docsEnum;
     this.reader = reader;
+    this.fieldSelector = fieldSelector;
   }
 
   @Override
@@ -66,7 +73,8 @@ public class TermDocIterable implements Iterable<Document> {
   }
 
   private Document getDoc() throws IOException {
-    return reader.document(docsEnum.docID());
+    reader.document(docsEnum.docID(), fieldSelector);
+    return fieldSelector.getDocument();
   }
 
   private boolean getNext() {
@@ -76,7 +84,7 @@ public class TermDocIterable implements Iterable<Document> {
         return false;
       }
       Bits liveDocs = MultiFields.getLiveDocs(reader);
-      if(liveDocs != null){
+      if (liveDocs != null) {
         while (!liveDocs.get(docsEnum.docID())) {
           next = docsEnum.nextDoc();
         }


Mime
View raw message