lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gsing...@apache.org
Subject svn commit: r636745 [2/2] - in /lucene/java/trunk: ./ contrib/instantiated/ contrib/instantiated/docs/ contrib/instantiated/src/ contrib/instantiated/src/java/ contrib/instantiated/src/java/org/ contrib/instantiated/src/java/org/apache/ contrib/instant...
Date Thu, 13 Mar 2008 12:34:37 GMT
Added: lucene/java/trunk/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermEnum.java?rev=636745&view=auto
==============================================================================
--- lucene/java/trunk/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermEnum.java
(added)
+++ lucene/java/trunk/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermEnum.java
Thu Mar 13 05:34:30 2008
@@ -0,0 +1,109 @@
+package org.apache.lucene.store.instantiated;
+
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermEnum;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+/**
+ * A {@link org.apache.lucene.index.TermEnum} navigating an {@link org.apache.lucene.store.instantiated.InstantiatedIndexReader}.
+ */
+public class InstantiatedTermEnum
+    extends TermEnum {
+
+  private final InstantiatedIndexReader reader;
+
+  public InstantiatedTermEnum(InstantiatedIndexReader reader) {
+    this.nextTermIndex = 0;
+    this.reader = reader;
+  }
+
+  public InstantiatedTermEnum(InstantiatedIndexReader reader, int startPosition) {
+    this.reader = reader;
+    this.nextTermIndex = startPosition;
+    next();
+  }
+
+  private int nextTermIndex;
+  private InstantiatedTerm term;
+
+  /**
+   * Increments the enumeration to the next element.  True if one exists.
+   */
+  public boolean next() {
+    if (reader.getIndex().getOrderedTerms().length <= nextTermIndex) {
+      return false;
+    } else {
+      term = reader.getIndex().getOrderedTerms()[nextTermIndex];
+      nextTermIndex++;
+      return true;
+    }
+  }
+
+  /**
+   * Returns the current Term in the enumeration.
+   */
+  public Term term() {
+    return /*term == null ? null :*/ term.getTerm();
+  }
+
+  /**
+   * Returns the docFreq of the current Term in the enumeration.
+   */
+  public int docFreq() {
+    return term.getAssociatedDocuments().length;
+  }
+
+  /**
+   * Closes the enumeration to further activity, freeing resources.
+   */
+  public void close() {
+  }
+
+
+  public boolean skipTo(Term target) throws IOException {
+
+    // this method is not known to be used by anything
+    // in lucene for many years now, so there is
+    // very to gain by optimizing this method more,
+
+    InstantiatedTerm term = reader.getIndex().findTerm(target);
+    if (term != null) {
+      this.term = term;
+      nextTermIndex = term.getTermIndex() + 1;
+      return true;
+    } else {
+      int pos = Arrays.binarySearch(reader.getIndex().getOrderedTerms(), target, InstantiatedTerm.termComparator);
+      if (pos < 0) {
+        pos = -1 - pos;
+      }
+
+      if (pos > reader.getIndex().getOrderedTerms().length) {
+        return false;
+      }
+      this.term = reader.getIndex().getOrderedTerms()[pos];
+      nextTermIndex = pos + 1;
+      return true;
+    }
+  }
+}
+
+
+

Propchange: lucene/java/trunk/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermEnum.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: lucene/java/trunk/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermFreqVector.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermFreqVector.java?rev=636745&view=auto
==============================================================================
--- lucene/java/trunk/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermFreqVector.java
(added)
+++ lucene/java/trunk/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermFreqVector.java
Thu Mar 13 05:34:30 2008
@@ -0,0 +1,112 @@
+package org.apache.lucene.store.instantiated;
+
+import org.apache.lucene.index.TermFreqVector;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Vector space view of a document in an {@link InstantiatedIndexReader}.
+ *
+ * @see org.apache.lucene.index.TermFreqVector
+ */
+public class InstantiatedTermFreqVector
+    implements TermFreqVector, Serializable {
+
+  private static final long serialVersionUID = 1l;
+
+  private final List<InstantiatedTermDocumentInformation> termDocumentInformations;
+  private final String field;
+  private final String terms[];
+  private final int termFrequencies[];
+
+  public InstantiatedTermFreqVector(InstantiatedDocument document, String field) {
+    this.field = field;
+    termDocumentInformations = document.getVectorSpace().get(field);
+    terms = new String[termDocumentInformations.size()];
+    termFrequencies = new int[termDocumentInformations.size()];
+
+    for (int i = 0; i < termDocumentInformations.size(); i++) {
+      InstantiatedTermDocumentInformation termDocumentInformation = termDocumentInformations.get(i);
+      terms[i] = termDocumentInformation.getTerm().text();
+      termFrequencies[i] = termDocumentInformation.getTermPositions().length;
+    }
+  }
+
+  /**
+   * @return The number of the field this vector is associated with
+   */
+  public String getField() {
+    return field;
+  }
+
+  public String toString() {
+    StringBuffer sb = new StringBuffer();
+    sb.append('{');
+    sb.append(field).append(": ");
+    if (terms != null) {
+      for (int i = 0; i < terms.length; i++) {
+        if (i > 0) sb.append(", ");
+        sb.append(terms[i]).append('/').append(termFrequencies[i]);
+      }
+    }
+    sb.append('}');
+
+    return sb.toString();
+  }
+
+  public int size() {
+    return terms == null ? 0 : terms.length;
+  }
+
+  public String[] getTerms() {
+    return terms;
+  }
+
+  public int[] getTermFrequencies() {
+    return termFrequencies;
+  }
+
+  public int indexOf(String termText) {
+    if (terms == null)
+      return -1;
+    int res = Arrays.binarySearch(terms, termText);
+    return res >= 0 ? res : -1;
+  }
+
+  public int[] indexesOf(String[] termNumbers, int start, int len) {
+    // TODO: there must be a more efficient way of doing this.
+    //       At least, we could advance the lower bound of the terms array
+    //       as we find valid indices. Also, it might be possible to leverage
+    //       this even more by starting in the middle of the termNumbers array
+    //       and thus dividing the terms array maybe in half with each found index.
+    int res[] = new int[len];
+
+    for (int i = 0; i < len; i++) {
+      res[i] = indexOf(termNumbers[start + i]);
+    }
+    return res;
+  }
+
+  public List<InstantiatedTermDocumentInformation> getTermDocumentInformations() {
+    return termDocumentInformations;
+  }
+
+}

Propchange: lucene/java/trunk/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermFreqVector.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: lucene/java/trunk/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermPositionVector.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermPositionVector.java?rev=636745&view=auto
==============================================================================
--- lucene/java/trunk/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermPositionVector.java
(added)
+++ lucene/java/trunk/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermPositionVector.java
Thu Mar 13 05:34:30 2008
@@ -0,0 +1,47 @@
+package org.apache.lucene.store.instantiated;
+
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.index.TermPositionVector;
+import org.apache.lucene.index.TermVectorOffsetInfo;
+
+import java.io.Serializable;
+
+/**
+ * Extended vector space view of a document in an {@link InstantiatedIndexReader}.
+ *
+ * @see org.apache.lucene.index.TermPositionVector
+ */
+public class InstantiatedTermPositionVector
+    extends InstantiatedTermFreqVector
+    implements TermPositionVector, Serializable {
+
+  private static final long serialVersionUID = 1l;
+
+  public InstantiatedTermPositionVector(InstantiatedDocument document, String field) {
+    super(document, field);
+  }
+
+  public int[] getTermPositions(int index) {
+    return getTermDocumentInformations().get(index).getTermPositions();
+  }
+
+  public TermVectorOffsetInfo[] getOffsets(int index) {
+    return getTermDocumentInformations().get(index).getTermOffsets();
+  }
+
+}

Propchange: lucene/java/trunk/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermPositionVector.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: lucene/java/trunk/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermPositions.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermPositions.java?rev=636745&view=auto
==============================================================================
--- lucene/java/trunk/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermPositions.java
(added)
+++ lucene/java/trunk/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermPositions.java
Thu Mar 13 05:34:30 2008
@@ -0,0 +1,100 @@
+package org.apache.lucene.store.instantiated;
+
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.index.TermPositions;
+
+import java.io.IOException;
+
+/**
+ * A {@link org.apache.lucene.index.TermPositions} navigating an {@link InstantiatedIndexReader}.
+ */
+public class InstantiatedTermPositions
+    extends InstantiatedTermDocs
+    implements TermPositions {
+
+  public int getPayloadLength() {
+    return currentDocumentInformation.getPayloads()[currentTermPositionIndex].length;
+  }
+
+  public byte[] getPayload(byte[] data, int offset) throws IOException {
+    byte[] payloads = currentDocumentInformation.getPayloads()[currentTermPositionIndex];
+
+    // read payloads lazily
+    if (data == null || data.length - offset < getPayloadLength()) {
+      // the array is too small to store the payload data,
+      return payloads;
+    } else {
+      System.arraycopy(payloads, 0, data, offset, payloads.length);
+      return data;
+    }
+  }
+
+  public boolean isPayloadAvailable() {
+    return currentDocumentInformation.getPayloads()[currentTermPositionIndex] != null;
+  }
+
+  public InstantiatedTermPositions(InstantiatedIndexReader reader) {
+    super(reader);
+  }
+
+  /**
+   * Returns next position in the current document.  It is an error to call
+   * this more than {@link #freq()} times
+   * without calling {@link #next()}<p> This is
+   * invalid until {@link #next()} is called for
+   * the first time.
+   */
+  public int nextPosition() {
+    currentTermPositionIndex++;
+    // if you get an array out of index exception here,
+    // it might be due to currentDocumentInformation.getIndexFromTerm not beeing set!!
+    return currentDocumentInformation.getTermPositions()[currentTermPositionIndex];
+  }
+
+  private int currentTermPositionIndex;
+
+  /**
+   * Moves to the next pair in the enumeration.
+   * <p> Returns true if there is such a next pair in the enumeration.
+   */
+  @Override
+  public boolean next() {
+    currentTermPositionIndex = -1;
+    return super.next();
+  }
+
+  /**
+   * Skips entries to the first beyond the current whose document number is
+   * greater than or equal to <currentTermPositionIndex>target</currentTermPositionIndex>.
<p>Returns true iff there is such
+   * an entry.  <p>Behaves as if written: <pre>
+   *   boolean skipTo(int target) {
+   *     do {
+   *       if (!next())
+   * 	     return false;
+   *     } while (target > doc());
+   *     return true;
+   *   }
+   * </pre>
+   * Some implementations are considerably more efficient than that.
+   */
+  @Override
+  public boolean skipTo(int target) {
+    currentTermPositionIndex = -1;
+    return super.skipTo(target);
+  }
+}

Propchange: lucene/java/trunk/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermPositions.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: lucene/java/trunk/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/package.html
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/package.html?rev=636745&view=auto
==============================================================================
--- lucene/java/trunk/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/package.html
(added)
+++ lucene/java/trunk/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/package.html
Thu Mar 13 05:34:30 2008
@@ -0,0 +1,90 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
+<html>
+<!--**
+* Copyright 2005 The Apache Software Foundation
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*
+-->
+<head>
+  <title>InstantiatedIndex</title>
+</head>
+<body>
+<p>WARNING: This contrib is experimental and the APIs may change without warning.</p>
+<h2>Abstract</h2>
+
+<p>
+  Represented as a coupled graph of class instances, this
+  all-in-memory index store implementation delivers search
+  results up to a 100 times faster than the file-centric RAMDirectory
+  at the cost of greater RAM consumption.
+</p>
+
+<h2>API</h2>
+
+<p>
+  Just as the default store implementation, InstantiatedIndex
+  comes with an IndexReader and IndexWriter. The latter share
+  many method signatures with the file-centric IndexWriter.
+</p>
+
+<p>
+  It is also possible to load the content of another index
+  by passing an IndexReader to the InstantiatedIndex constructor.
+</p>
+
+<h2>Performance</h2>
+
+<p>
+  At a few thousand ~160 characters long documents
+  InstantiaedIndex outperforms RAMDirectory some 50x,
+  15x at 100 documents of 2000 charachters length,
+  and is linear to RAMDirectory at 10,000 documents of 2000 characters length.
+</p>
+
+<p>Mileage may vary depending on term saturation.</p>
+
+<p>
+  Populated with a single document InstantiatedIndex is almost, but not quite, as fast as
MemoryIndex.    
+</p>
+
+<p>
+  It takes more or less the same time to populate an InstantiatedIndex
+  as it takes to populate a RAMDirectory. Hardly any effort has been put
+  in to optimizing the InstantiatedIndexWriter, only minimizing the amount
+  of time needed to write-lock the index has been considered.
+</p>
+
+<h2>Caveats</h2>
+<ul>
+  <li>No locks! Consider using InstantiatedIndex as if it was immutable.</li>
+  <li>No documents with fields containing readers!</li>
+  <li>Only FieldOption.All allowed by IndexReader#getFieldNames(FieldOption).</li>
+  <li>No field selection when retrieving documents, as all stored field are available
in memory.</li>
+</ul>
+
+<h2>Use cases</h2>
+
+<p>
+  Could replace any small index that could do with greater response time.
+  spell check a priori index,
+  the index of new documents exposed to user search agent queries,
+  to compile classifiers in machine learning environments, et c.
+</p>
+
+<h2>Class diagram</h2>
+<a href="../../../../../../../docs/classdiagram.png"><img width="640px" height="480px"
src="../../../../../../../docs/classdiagram.png" alt="class diagram"></a>
+<br/>
+<a href="../../../../../../../docs/classdiagram.uxf">Diagram</a> rendered using
<a href="http://umlet.com">UMLet</a> 7.1.
+</body>
+</html>

Propchange: lucene/java/trunk/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/package.html
------------------------------------------------------------------------------
    svn:eol-style = native

Added: lucene/java/trunk/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java?rev=636745&view=auto
==============================================================================
--- lucene/java/trunk/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
(added)
+++ lucene/java/trunk/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
Thu Mar 13 05:34:30 2008
@@ -0,0 +1,424 @@
+package org.apache.lucene.store.instantiated;
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import junit.framework.TestCase;
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.*;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.RAMDirectory;
+
+import java.io.IOException;
+import java.util.*;
+
+/**
+ * Asserts equality of content and behaviour of two index readers.
+ */
+public class TestIndicesEquals extends TestCase {
+
+//  public void test2() throws Exception {
+//    FSDirectory fsdir = FSDirectory.getDirectory("/tmp/fatcorpus");
+//    IndexReader ir = IndexReader.open(fsdir);
+//    InstantiatedIndex ii = new InstantiatedIndex(ir);
+//    ir.close();
+//    testEquals(fsdir, ii);
+//  }
+
+
+  public void testLoadIndexReader() throws Exception {
+    RAMDirectory dir = new RAMDirectory();
+
+    // create dir data
+    IndexWriter indexWriter = new IndexWriter(dir, new StandardAnalyzer(), true);
+    for (int i = 0; i < 5; i++) {
+      Document document = new Document();
+      assembleDocument(document, i);
+      indexWriter.addDocument(document);
+    }
+    indexWriter.close();
+
+    // test load ii from index reader
+    IndexReader ir = IndexReader.open(dir);
+    InstantiatedIndex ii = new InstantiatedIndex(ir);
+    ir.close();
+
+    testEquals(dir, ii);
+  }
+
+  public void testInstantiatedIndexWriter() throws Exception {
+
+
+    RAMDirectory dir = new RAMDirectory();
+    InstantiatedIndex ii = new InstantiatedIndex();
+
+    // create dir data
+    IndexWriter indexWriter = new IndexWriter(dir, new StandardAnalyzer(), true);
+    for (int i = 0; i < 500; i++) {
+      Document document = new Document();
+      assembleDocument(document, i);
+      indexWriter.addDocument(document);
+    }
+    indexWriter.close();
+
+    // test ii writer
+    InstantiatedIndexWriter instantiatedIndexWriter = ii.indexWriterFactory(new StandardAnalyzer(),
true);
+    for (int i = 0; i < 500; i++) {
+      Document document = new Document();
+      assembleDocument(document, i);
+      instantiatedIndexWriter.addDocument(document);
+    }
+    instantiatedIndexWriter.close();
+
+    testEquals(dir, ii);
+
+    testTermDocs(dir, ii);
+
+
+  }
+
+
+  private void testTermDocs(Directory aprioriIndex, InstantiatedIndex testIndex) throws Exception
{
+
+    IndexReader aprioriReader = IndexReader.open(aprioriIndex);
+    IndexReader testReader = testIndex.indexReaderFactory();
+
+    TermEnum aprioriTermEnum = aprioriReader.terms(new Term("c", "danny"));
+
+    TermDocs aprioriTermDocs = aprioriReader.termDocs(aprioriTermEnum.term());
+    TermDocs testTermDocs = testReader.termDocs(aprioriTermEnum.term());
+
+    assertEquals(aprioriTermDocs.next(), testTermDocs.next());
+    assertEquals(aprioriTermDocs.doc(), testTermDocs.doc());
+
+    assertEquals(aprioriTermDocs.skipTo(100), testTermDocs.skipTo(100));
+    assertEquals(aprioriTermDocs.doc(), testTermDocs.doc());
+
+    assertEquals(aprioriTermDocs.next(), testTermDocs.next());
+    assertEquals(aprioriTermDocs.doc(), testTermDocs.doc());
+
+    assertEquals(aprioriTermDocs.next(), testTermDocs.next());
+    assertEquals(aprioriTermDocs.doc(), testTermDocs.doc());
+
+    assertEquals(aprioriTermDocs.skipTo(110), testTermDocs.skipTo(110));
+    assertEquals(aprioriTermDocs.doc(), testTermDocs.doc());
+
+    assertEquals(aprioriTermDocs.skipTo(10), testTermDocs.skipTo(10));
+    assertEquals(aprioriTermDocs.doc(), testTermDocs.doc());
+
+    assertEquals(aprioriTermDocs.skipTo(210), testTermDocs.skipTo(210));
+    assertEquals(aprioriTermDocs.doc(), testTermDocs.doc());
+
+    aprioriTermDocs.close();
+    aprioriReader.close();
+
+    testTermDocs.close();
+    testReader.close();
+
+  }
+
+  private void assembleDocument(Document document, int i) {
+    document.add(new Field("a", i + " Do you really want to go and live in that house all
winter?", Field.Store.YES, Field.Index.TOKENIZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+    if (i > 0) {
+      document.add(new Field("b0", i + " All work and no play makes Jack a dull boy", Field.Store.YES,
Field.Index.TOKENIZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+      document.add(new Field("b1", i + " All work and no play makes Jack a dull boy", Field.Store.YES,
Field.Index.NO_NORMS, Field.TermVector.NO));
+      document.add(new Field("b2", i + " All work and no play makes Jack a dull boy", Field.Store.NO,
Field.Index.UN_TOKENIZED, Field.TermVector.NO));
+      document.add(new Field("b3", i + " All work and no play makes Jack a dull boy", Field.Store.YES,
Field.Index.NO, Field.TermVector.NO));
+      if (i > 1) {
+        document.add(new Field("c", i + " Redrum redrum", Field.Store.YES, Field.Index.TOKENIZED,
Field.TermVector.WITH_POSITIONS_OFFSETS));
+        if (i > 2) {
+          document.add(new Field("d", i + " Hello Danny, come and play with us... forever
and ever. and ever.", Field.Store.YES, Field.Index.TOKENIZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+          if (i > 3) {
+            Field f = new Field("e", i + " Heres Johnny!", Field.Store.YES, Field.Index.TOKENIZED,
Field.TermVector.WITH_POSITIONS_OFFSETS);
+            f.setOmitNorms(true);
+            document.add(f);
+            if (i > 4) {
+              final List<Token> tokens = new ArrayList<Token>(2);
+              Token t = new Token("the", 0, 2, "text");
+              t.setPayload(new Payload(new byte[]{1, 2, 3}));
+              tokens.add(t);
+              t = new Token("end", 3, 5, "text");
+              t.setPayload(new Payload(new byte[]{2}));
+              tokens.add(t);
+              tokens.add(new Token("fin", 7, 9));
+              document.add(new Field("f", new TokenStream() {
+                Iterator<Token> it = tokens.iterator();
+
+                public Token next() throws IOException {
+                  if (!it.hasNext()) {
+                    return null;
+                  }
+                  return it.next();
+                }
+
+                public void reset() throws IOException {
+                  it = tokens.iterator();
+                }
+              }));
+            }
+          }
+        }
+      }
+    }
+  }
+
+
+  /**
+   * Asserts that the content of two index readers equal each other.
+   *
+   * @param aprioriIndex the index that is known to be correct
+   * @param testIndex    the index that is supposed to equals the apriori index.
+   * @throws Exception
+   */
+  protected void testEquals(Directory aprioriIndex, InstantiatedIndex testIndex) throws Exception
{
+
+    IndexReader aprioriReader = IndexReader.open(aprioriIndex);
+    IndexReader testReader = testIndex.indexReaderFactory();
+
+    assertEquals(aprioriReader.numDocs(), testReader.numDocs());
+
+    for (Object field : aprioriReader.getFieldNames(IndexReader.FieldOption.ALL)) {
+
+      // test norms as used by normal use
+
+      byte[] aprioriNorms = aprioriReader.norms((String) field);
+      byte[] testNorms = testReader.norms((String) field);
+
+      assertEquals(aprioriNorms.length, testNorms.length);
+
+      for (int i = 0; i < aprioriNorms.length; i++) {
+        assertEquals("norms does not equals for field " + field + " in document " + i, aprioriNorms[i],
testNorms[i]);
+      }
+
+      // test norms as used by multireader
+
+      aprioriNorms = new byte[aprioriReader.maxDoc()];
+      aprioriReader.norms((String) field, aprioriNorms, 0);
+
+      testNorms = new byte[testReader.maxDoc()];
+      testReader.norms((String) field, testNorms, 0);
+
+      assertEquals(aprioriNorms.length, testNorms.length);
+
+      for (int i = 0; i < aprioriNorms.length; i++) {
+        assertEquals("norms does not equals for field " + field + " in document " + i, aprioriNorms[i],
testNorms[i]);
+      }
+
+
+    }
+
+    for (int docIndex = 0; docIndex < aprioriReader.numDocs(); docIndex++) {
+      assertEquals(aprioriReader.isDeleted(docIndex), testReader.isDeleted(docIndex));
+    }
+
+    // compare term enumeration stepping
+
+    TermEnum aprioriTermEnum = aprioriReader.terms();
+    TermEnum testTermEnum = testReader.terms();
+
+
+    while (true) {
+
+      if (!aprioriTermEnum.next()) {
+        assertFalse(testTermEnum.next());
+        break;
+      }
+      assertTrue(testTermEnum.next());
+
+      assertEquals(aprioriTermEnum.term(), testTermEnum.term());
+      assertTrue(aprioriTermEnum.docFreq() == testTermEnum.docFreq());
+
+      // compare termDocs seeking
+
+      TermDocs aprioriTermDocsSeeker = aprioriReader.termDocs(aprioriTermEnum.term());
+      TermDocs testTermDocsSeeker = testReader.termDocs(testTermEnum.term());
+
+      while (aprioriTermDocsSeeker.next()) {
+        assertTrue(testTermDocsSeeker.skipTo(aprioriTermDocsSeeker.doc()));
+        assertEquals(aprioriTermDocsSeeker.doc(), testTermDocsSeeker.doc());
+      }
+
+      aprioriTermDocsSeeker.close();
+      testTermDocsSeeker.close();
+
+      // compare documents per term
+
+      assertEquals(aprioriReader.docFreq(aprioriTermEnum.term()), testReader.docFreq(testTermEnum.term()));
+
+      TermDocs aprioriTermDocs = aprioriReader.termDocs(aprioriTermEnum.term());
+      TermDocs testTermDocs = testReader.termDocs(testTermEnum.term());
+
+      while (true) {
+        if (!aprioriTermDocs.next()) {
+          assertFalse(testTermDocs.next());
+          break;
+        }
+        assertTrue(testTermDocs.next());
+
+        assertEquals(aprioriTermDocs.doc(), testTermDocs.doc());
+        assertEquals(aprioriTermDocs.freq(), testTermDocs.freq());
+      }
+
+      aprioriTermDocs.close();
+      testTermDocs.close();
+
+      // compare term positions
+
+      TermPositions testTermPositions = testReader.termPositions(testTermEnum.term());
+      TermPositions aprioriTermPositions = aprioriReader.termPositions(aprioriTermEnum.term());
+
+      if (aprioriTermPositions != null) {
+
+        for (int docIndex = 0; docIndex < aprioriReader.maxDoc(); docIndex++) {
+          boolean hasNext = aprioriTermPositions.next();
+          if (hasNext) {
+            assertTrue(testTermPositions.next());
+
+            assertEquals(aprioriTermPositions.freq(), testTermPositions.freq());
+
+
+            for (int termPositionIndex = 0; termPositionIndex < aprioriTermPositions.freq();
termPositionIndex++) {
+              int aprioriPos = aprioriTermPositions.nextPosition();
+              int testPos = testTermPositions.nextPosition();
+
+              if (aprioriPos != testPos) {
+                assertEquals(aprioriPos, testPos);
+              }
+
+
+              assertEquals(aprioriTermPositions.isPayloadAvailable(), testTermPositions.isPayloadAvailable());
+              if (aprioriTermPositions.isPayloadAvailable()) {
+                assertEquals(aprioriTermPositions.getPayloadLength(), testTermPositions.getPayloadLength());
+                byte[] aprioriPayloads = aprioriTermPositions.getPayload(new byte[aprioriTermPositions.getPayloadLength()],
0);
+                byte[] testPayloads = testTermPositions.getPayload(new byte[testTermPositions.getPayloadLength()],
0);
+                for (int i = 0; i < aprioriPayloads.length; i++) {
+                  assertEquals(aprioriPayloads[i], testPayloads[i]);
+                }
+              }
+
+            }
+          }
+        }
+
+        aprioriTermPositions.close();
+        testTermPositions.close();
+
+      }
+    }
+
+    // compare term enumeration seeking
+
+    aprioriTermEnum = aprioriReader.terms();
+
+    TermEnum aprioriTermEnumSeeker = aprioriReader.terms();
+    TermEnum testTermEnumSeeker = testReader.terms();
+
+    while (aprioriTermEnum.next()) {
+      if (aprioriTermEnumSeeker.skipTo(aprioriTermEnum.term())) {
+        assertTrue(testTermEnumSeeker.skipTo(aprioriTermEnum.term()));
+        assertEquals(aprioriTermEnumSeeker.term(), testTermEnumSeeker.term());
+      } else {
+        assertFalse(testTermEnumSeeker.skipTo(aprioriTermEnum.term()));
+      }
+    }
+
+    aprioriTermEnum.close();
+    aprioriTermEnumSeeker.close();
+    testTermEnumSeeker.close();
+
+    // skip to non existing terms
+
+    aprioriTermEnumSeeker = aprioriReader.terms();
+    testTermEnumSeeker = testReader.terms();
+
+    aprioriTermEnum = aprioriReader.terms();
+    aprioriTermEnum.next();
+    Term nonExistingTerm = new Term(aprioriTermEnum.term().field(), "bzzzzoo993djdj380sdf");
+    aprioriTermEnum.close();
+
+    assertEquals(aprioriTermEnumSeeker.skipTo(nonExistingTerm), testTermEnumSeeker.skipTo(nonExistingTerm));
+    assertEquals(aprioriTermEnumSeeker.term(), testTermEnumSeeker.term());
+
+    aprioriTermEnumSeeker.close();
+    testTermEnumSeeker.close();
+
+    // compare term vectors and position vectors
+
+    for (int documentNumber = 0; documentNumber < aprioriReader.numDocs(); documentNumber++)
{
+
+      if (documentNumber > 0) {
+        assertNotNull(aprioriReader.getTermFreqVector(documentNumber, "b0"));
+        assertNull(aprioriReader.getTermFreqVector(documentNumber, "b1"));
+
+        assertNotNull(testReader.getTermFreqVector(documentNumber, "b0"));
+        assertNull(testReader.getTermFreqVector(documentNumber, "b1"));
+
+      }
+
+      TermFreqVector[] aprioriFreqVectors = aprioriReader.getTermFreqVectors(documentNumber);
+      TermFreqVector[] testFreqVectors = testReader.getTermFreqVectors(documentNumber);
+
+      if (aprioriFreqVectors != null && testFreqVectors != null) {
+
+        Arrays.sort(aprioriFreqVectors, new Comparator<TermFreqVector>() {
+          public int compare(TermFreqVector termFreqVector, TermFreqVector termFreqVector1)
{
+            return termFreqVector.getField().compareTo(termFreqVector1.getField());
+          }
+        });
+        Arrays.sort(testFreqVectors, new Comparator<TermFreqVector>() {
+          public int compare(TermFreqVector termFreqVector, TermFreqVector termFreqVector1)
{
+            return termFreqVector.getField().compareTo(termFreqVector1.getField());
+          }
+        });
+
+        assertEquals("document " + documentNumber + " vectors does not match", aprioriFreqVectors.length,
testFreqVectors.length);
+
+        for (int freqVectorIndex = 0; freqVectorIndex < aprioriFreqVectors.length; freqVectorIndex++)
{
+          assertTrue(Arrays.equals(aprioriFreqVectors[freqVectorIndex].getTermFrequencies(),
testFreqVectors[freqVectorIndex].getTermFrequencies()));
+          assertTrue(Arrays.equals(aprioriFreqVectors[freqVectorIndex].getTerms(), testFreqVectors[freqVectorIndex].getTerms()));
+
+          if (aprioriFreqVectors[freqVectorIndex] instanceof TermPositionVector) {
+            TermPositionVector aprioriTermPositionVector = (TermPositionVector) aprioriFreqVectors[freqVectorIndex];
+            TermPositionVector testTermPositionVector = (TermPositionVector) testFreqVectors[freqVectorIndex];
+
+            for (int positionVectorIndex = 0; positionVectorIndex < aprioriFreqVectors[freqVectorIndex].getTerms().length;
positionVectorIndex++)
+            {
+              if (aprioriTermPositionVector.getOffsets(positionVectorIndex) != null) {
+                assertTrue(Arrays.equals(aprioriTermPositionVector.getOffsets(positionVectorIndex),
testTermPositionVector.getOffsets(positionVectorIndex)));
+              }
+
+              if (aprioriTermPositionVector.getTermPositions(positionVectorIndex) != null)
{
+                assertTrue(Arrays.equals(aprioriTermPositionVector.getTermPositions(positionVectorIndex),
testTermPositionVector.getTermPositions(positionVectorIndex)));
+              }
+            }
+          }
+
+        }
+      }
+
+    }
+
+    aprioriTermEnum.close();
+    testTermEnum.close();
+
+    aprioriReader.close();
+    testReader.close();
+  }
+
+}

Propchange: lucene/java/trunk/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
------------------------------------------------------------------------------
    svn:eol-style = native



Mime
View raw message