lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From markrmil...@apache.org
Subject svn commit: r831094 [8/8] - in /lucene/java/branches/flex_1458: ./ contrib/ contrib/analyzers/common/src/java/org/apache/lucene/analysis/br/ contrib/analyzers/common/src/java/org/apache/lucene/analysis/cz/ contrib/analyzers/common/src/java/org/apache/l...
Date Thu, 29 Oct 2009 21:29:56 GMT
Propchange: lucene/java/branches/flex_1458/src/test/org/apache/lucene/document/TestNumberTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Oct 29 21:29:47 2009
@@ -1,3 +1,3 @@
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestNumberTools.java:748824
-/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestNumberTools.java:829134
-/lucene/java/trunk/src/test/org/apache/lucene/document/TestNumberTools.java:824912-825292,826213-829438
+/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestNumberTools.java:829134,829881
+/lucene/java/trunk/src/test/org/apache/lucene/document/TestNumberTools.java:824912-825292,826213-830804

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=831094&r1=831093&r2=831094&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
(original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
Thu Oct 29 21:29:47 2009
@@ -22,21 +22,28 @@
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
 import java.io.OutputStream;
 import java.util.Arrays;
 import java.util.Enumeration;
 import java.util.List;
+import java.util.ArrayList;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
 
 import org.apache.lucene.analysis.WhitespaceAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.document.FieldSelector;
+import org.apache.lucene.document.FieldSelectorResult;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.ReaderUtil;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util._TestUtil;
 
@@ -127,22 +134,105 @@
                              "23.nocfs",
                              "24.cfs",
                              "24.nocfs",
+                             "29.cfs",
+                             "29.nocfs",
   };
+  
+  private void assertCompressedFields29(Directory dir, boolean shouldStillBeCompressed) throws
IOException {
+    int count = 0;
+    final int TEXT_PLAIN_LENGTH = TEXT_TO_COMPRESS.length() * 2;
+    // FieldSelectorResult.SIZE returns 2*number_of_chars for String fields:
+    final int BINARY_PLAIN_LENGTH = BINARY_TO_COMPRESS.length;
+    
+    IndexReader reader = IndexReader.open(dir, true);
+    try {
+      // look into sub readers and check if raw merge is on/off
+      List<IndexReader> readers = new ArrayList<IndexReader>();
+      ReaderUtil.gatherSubReaders(readers, reader);
+      for (IndexReader ir : readers) {
+        final FieldsReader fr = ((SegmentReader) ir).getFieldsReader();
+        assertTrue("for a 2.9 index, FieldsReader.canReadRawDocs() must be false and other
way round for a trunk index",
+          shouldStillBeCompressed != fr.canReadRawDocs());
+      }
+    
+      // test that decompression works correctly
+      for(int i=0; i<reader.maxDoc(); i++) {
+        if (!reader.isDeleted(i)) {
+          Document d = reader.document(i);
+          if (d.get("content3") != null) continue;
+          count++;
+          Fieldable compressed = d.getFieldable("compressed");
+          if (Integer.parseInt(d.get("id")) % 2 == 0) {
+            assertFalse(compressed.isBinary());
+            assertEquals("incorrectly decompressed string", TEXT_TO_COMPRESS, compressed.stringValue());
+          } else {
+            assertTrue(compressed.isBinary());
+            assertTrue("incorrectly decompressed binary", Arrays.equals(BINARY_TO_COMPRESS,
compressed.getBinaryValue()));
+          }
+        }
+      }
+      
+      // check if field was decompressed after optimize
+      for(int i=0; i<reader.maxDoc(); i++) {
+        if (!reader.isDeleted(i)) {
+          Document d = reader.document(i, new FieldSelector() {
+            public FieldSelectorResult accept(String fieldName) {
+              return ("compressed".equals(fieldName)) ? FieldSelectorResult.SIZE : FieldSelectorResult.LOAD;
+            }
+          });
+          if (d.get("content3") != null) continue;
+          count++;
+          // read the size from the binary value using DataInputStream (this prevents us
from doing the shift ops ourselves):
+          final DataInputStream ds = new DataInputStream(new ByteArrayInputStream(d.getFieldable("compressed").getBinaryValue()));
+          final int actualSize = ds.readInt();
+          ds.close();
+          final int compressedSize = Integer.parseInt(d.get("compressedSize"));
+          final boolean binary = Integer.parseInt(d.get("id")) % 2 > 0;
+          final int shouldSize = shouldStillBeCompressed ?
+            compressedSize :
+            (binary ? BINARY_PLAIN_LENGTH : TEXT_PLAIN_LENGTH);
+          assertEquals("size incorrect", shouldSize, actualSize);
+          if (!shouldStillBeCompressed) {
+            assertFalse("uncompressed field should have another size than recorded in index",
compressedSize == actualSize);
+          }
+        }
+      }
+      assertEquals("correct number of tests", 34 * 2, count);
+    } finally {
+      reader.close();
+    }
+  }
 
   public void testOptimizeOldIndex() throws IOException {
+    int hasTested29 = 0;
+    
     for(int i=0;i<oldNames.length;i++) {
       String dirName = "src/test/org/apache/lucene/index/index." + oldNames[i];
       unzip(dirName, oldNames[i]);
       String fullPath = fullDir(oldNames[i]);
       Directory dir = FSDirectory.open(new File(fullPath));
+
+      if (oldNames[i].startsWith("29.")) {
+        assertCompressedFields29(dir, true);
+        hasTested29++;
+      }
+
       IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED);
       w.optimize();
       w.close();
 
       _TestUtil.checkIndex(dir);
+      
+      if (oldNames[i].startsWith("29.")) {
+        assertCompressedFields29(dir, false);
+        hasTested29++;
+      }
+
       dir.close();
       rmDir(oldNames[i]);
     }
+    
+    assertEquals("test for compressed field should have run 4 times", 4, hasTested29);
   }
 
   public void testSearchOldIndex() throws IOException {
@@ -203,7 +293,8 @@
             !oldName.startsWith("22.")) {
 
           if (d.getField("content3") == null) {
-            assertEquals(5, fields.size());
+            final int numFields = oldName.startsWith("29.") ? 7 : 5;
+            assertEquals(numFields, fields.size());
             Field f = (Field) d.getField("id");
             assertEquals(""+i, f.stringValue());
 
@@ -497,6 +588,15 @@
     doc.add(new Field("utf8", "Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd",
Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
     doc.add(new Field("content2", "here is more content with aaa aaa aaa", Field.Store.YES,
Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
     doc.add(new Field("fie\u2C77ld", "field with non-ascii name", Field.Store.YES, Field.Index.ANALYZED,
Field.TermVector.WITH_POSITIONS_OFFSETS));
+    /* This was used in 2.9 to generate an index with compressed field:
+    if (id % 2 == 0) {
+      doc.add(new Field("compressed", TEXT_TO_COMPRESS, Field.Store.COMPRESS, Field.Index.NOT_ANALYZED));
+      doc.add(new Field("compressedSize", Integer.toString(TEXT_COMPRESSED_LENGTH), Field.Store.YES,
Field.Index.NOT_ANALYZED));
+    } else {
+      doc.add(new Field("compressed", BINARY_TO_COMPRESS, Field.Store.COMPRESS));    
+      doc.add(new Field("compressedSize", Integer.toString(BINARY_COMPRESSED_LENGTH), Field.Store.YES,
Field.Index.NOT_ANALYZED));
+    }
+    */
     writer.addDocument(doc);
   }
 
@@ -527,4 +627,22 @@
   public static String fullDir(String dirName) throws IOException {
     return new File(System.getProperty("tempDir"), dirName).getCanonicalPath();
   }
+
+  static final String TEXT_TO_COMPRESS = "this is a compressed field and should appear in
3.0 as an uncompressed field after merge";
+  // FieldSelectorResult.SIZE returns compressed size for compressed fields, which are internally
handled as binary;
+  // do it in the same way like FieldsWriter, do not use CompressionTools.compressString()
for compressed fields:
+  /* This was used in 2.9 to generate an index with compressed field:
+  static final int TEXT_COMPRESSED_LENGTH;
+  static {
+    try {
+      TEXT_COMPRESSED_LENGTH = CompressionTools.compress(TEXT_TO_COMPRESS.getBytes("UTF-8")).length;
+    } catch (Exception e) {
+      throw new RuntimeException();
+    }
+  }
+  */
+  static final byte[] BINARY_TO_COMPRESS = new byte[]{1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20};
+  /* This was used in 2.9 to generate an index with compressed field:
+  static final int BINARY_COMPRESSED_LENGTH = CompressionTools.compress(BINARY_TO_COMPRESS).length;
+  */
 }

Propchange: lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Oct 29 21:29:47 2009
@@ -1,3 +1,3 @@
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:748824
-/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:829134
-/lucene/java/trunk/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:824912-825292,826213-829438
+/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:829134,829881
+/lucene/java/trunk/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:824912-825292,826213-830804

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestSegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestSegmentMerger.java?rev=831094&r1=831093&r2=831094&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestSegmentMerger.java
(original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestSegmentMerger.java
Thu Oct 29 21:29:47 2009
@@ -18,6 +18,7 @@
  */
 
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.store.BufferedIndexInput;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.document.Document;
@@ -69,8 +70,9 @@
     merger.closeReaders();
     assertTrue(docsMerged == 2);
     //Should be able to open a new SegmentReader against the new directory
-    SegmentReader mergedReader = SegmentReader.get(new SegmentInfo(mergedSegment, docsMerged,
mergedDir, false, true,
-                                                                   -1, null, false, merger.hasProx(),
merger.getCodec()));
+    SegmentReader mergedReader = SegmentReader.get(false, mergedDir, new SegmentInfo(mergedSegment,
docsMerged, mergedDir, false, true,
+        -1, null, false, merger.hasProx(), merger.getCodec()), BufferedIndexInput.BUFFER_SIZE,
true, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, null);
+
     assertTrue(mergedReader != null);
     assertTrue(mergedReader.numDocs() == 2);
     Document newDoc1 = mergedReader.document(0);

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/JustCompileSearch.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/JustCompileSearch.java?rev=831094&r1=831093&r2=831094&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/JustCompileSearch.java
(original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/JustCompileSearch.java
Thu Oct 29 21:29:47 2009
@@ -309,10 +309,6 @@
       throw new UnsupportedOperationException(UNSUPPORTED_MSG);
     }
     
-    public Explanation explain(int doc) throws IOException {
-      throw new UnsupportedOperationException(UNSUPPORTED_MSG);
-    }
-
     public float score() throws IOException {
       throw new UnsupportedOperationException(UNSUPPORTED_MSG);
     }

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestBooleanScorer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestBooleanScorer.java?rev=831094&r1=831093&r2=831094&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestBooleanScorer.java
(original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestBooleanScorer.java
Thu Oct 29 21:29:47 2009
@@ -80,15 +80,14 @@
     Similarity sim = Similarity.getDefault();
     Scorer[] scorers = new Scorer[] {new Scorer(sim) {
       private int doc = -1;
-      public Explanation explain(int doc) throws IOException { return null; }
-      public float score() throws IOException { return 0; }
-      public int docID() { return doc; }
+      @Override public float score() throws IOException { return 0; }
+      @Override public int docID() { return doc; }
       
-      public int nextDoc() throws IOException {
+      @Override public int nextDoc() throws IOException {
         return doc = doc == -1 ? 3000 : NO_MORE_DOCS;
       }
 
-      public int advance(int target) throws IOException {
+      @Override public int advance(int target) throws IOException {
         return doc = target <= 3000 ? 3000 : NO_MORE_DOCS;
       }
       

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestPositiveScoresOnlyCollector.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestPositiveScoresOnlyCollector.java?rev=831094&r1=831093&r2=831094&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestPositiveScoresOnlyCollector.java
(original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestPositiveScoresOnlyCollector.java
Thu Oct 29 21:29:47 2009
@@ -30,19 +30,17 @@
       super(null);
     }
     
-    public Explanation explain(int doc) throws IOException { return null; }
-
-    public float score() throws IOException {
+    @Override public float score() throws IOException {
       return idx == scores.length ? Float.NaN : scores[idx];
     }
 
-    public int docID() { return idx; }
+    @Override public int docID() { return idx; }
 
-    public int nextDoc() throws IOException {
+    @Override public int nextDoc() throws IOException {
       return ++idx != scores.length ? idx : NO_MORE_DOCS;
     }
     
-    public int advance(int target) throws IOException {
+    @Override public int advance(int target) throws IOException {
       idx = target;
       return idx < scores.length ? idx : NO_MORE_DOCS;
     }

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestScoreCachingWrappingScorer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestScoreCachingWrappingScorer.java?rev=831094&r1=831093&r2=831094&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestScoreCachingWrappingScorer.java
(original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestScoreCachingWrappingScorer.java
Thu Oct 29 21:29:47 2009
@@ -32,9 +32,7 @@
       super(null);
     }
     
-    public Explanation explain(int doc) throws IOException { return null; }
-
-    public float score() throws IOException {
+    @Override public float score() throws IOException {
       // advance idx on purpose, so that consecutive calls to score will get
       // different results. This is to emulate computation of a score. If
       // ScoreCachingWrappingScorer is used, this should not be called more than
@@ -42,13 +40,13 @@
       return idx == scores.length ? Float.NaN : scores[idx++];
     }
 
-    public int docID() { return doc; }
+    @Override public int docID() { return doc; }
 
-    public int nextDoc() throws IOException {
+    @Override public int nextDoc() throws IOException {
       return ++doc < scores.length ? doc : NO_MORE_DOCS;
     }
     
-    public int advance(int target) throws IOException {
+    @Override public int advance(int target) throws IOException {
       doc = target;
       return doc < scores.length ? doc : NO_MORE_DOCS;
     }
@@ -65,7 +63,7 @@
       mscores = new float[numToCollect];
     }
     
-    public void collect(int doc) throws IOException {
+    @Override public void collect(int doc) throws IOException {
       // just a sanity check to avoid IOOB.
       if (idx == mscores.length) {
         return; 
@@ -78,15 +76,15 @@
       ++idx;
     }
 
-    public void setNextReader(IndexReader reader, int docBase)
+    @Override public void setNextReader(IndexReader reader, int docBase)
         throws IOException {
     }
 
-    public void setScorer(Scorer scorer) throws IOException {
+    @Override public void setScorer(Scorer scorer) throws IOException {
       this.scorer = new ScoreCachingWrappingScorer(scorer);
     }
     
-    public boolean acceptsDocsOutOfOrder() {
+    @Override public boolean acceptsDocsOutOfOrder() {
       return true;
     }
 

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestTermScorer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestTermScorer.java?rev=831094&r1=831093&r2=831094&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestTermScorer.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestTermScorer.java Thu
Oct 29 21:29:47 2009
@@ -155,48 +155,6 @@
         assertTrue("doc should be number 5", ts.docID() == 5);
     }
 
-    public void testExplain() throws Exception
-    {
-        Term allTerm = new Term(FIELD, "all");
-        TermQuery termQuery = new TermQuery(allTerm);
-
-        Weight weight = termQuery.weight(indexSearcher);
-
-        Scorer ts = weight.scorer(indexSearcher.getIndexReader(),
-                                  true, true);
-        Explanation explanation = ts.explain(0);
-        assertTrue("explanation is null and it shouldn't be", explanation != null);
-        //System.out.println("Explanation: " + explanation.toString());
-        //All this Explain does is return the term frequency
-        assertTrue("term frq is not 1", explanation.getValue() == 1);
-        explanation = ts.explain(1);
-        assertTrue("explanation is null and it shouldn't be", explanation != null);
-        //System.out.println("Explanation: " + explanation.toString());
-        //All this Explain does is return the term frequency
-        assertTrue("term frq is not 0", explanation.getValue() == 0);
-
-        Term dogsTerm = new Term(FIELD, "dogs");
-        termQuery = new TermQuery(dogsTerm);
-        weight = termQuery.weight(indexSearcher);
-
-        ts = weight.scorer(indexSearcher.getIndexReader(),
-                           true, true);
-        explanation = ts.explain(1);
-        assertTrue("explanation is null and it shouldn't be", explanation != null);
-        //System.out.println("Explanation: " + explanation.toString());
-        //All this Explain does is return the term frequency
-        float sqrtTwo = (float)Math.sqrt(2.0f);
-        assertTrue("term frq: " + explanation.getValue() + " is not the square root of 2",
explanation.getValue() == sqrtTwo);
-
-        explanation = ts.explain(10);//try a doc out of range
-        assertTrue("explanation is null and it shouldn't be", explanation != null);
-        //System.out.println("Explanation: " + explanation.toString());
-        //All this Explain does is return the term frequency
-
-        assertTrue("term frq: " + explanation.getValue() + " is not 0", explanation.getValue()
== 0);
-
-    }
-
     private class TestHit {
         public int doc;
         public float score;

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java?rev=831094&r1=831093&r2=831094&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java
(original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java
Thu Oct 29 21:29:47 2009
@@ -169,9 +169,7 @@
    */
   public void testSpanNearScorerExplain() throws Exception {
     SpanNearQuery q = makeQuery();
-    Weight w = q.weight(searcher);
-    Scorer s = w.scorer(searcher.getIndexReader(), true, false);
-    Explanation e = s.explain(1);
+    Explanation e = q.weight(searcher).explain(searcher.getIndexReader(), 1);
     assertTrue("Scorer explanation value for doc#1 isn't positive: "
                + e.toString(),
                0.0f < e.getValue());



Mime
View raw message