lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r1239205 - in /lucene/dev/trunk/lucene/src: java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java test-framework/java/org/apache/lucene/index/RandomCodec.java test/org/apache/lucene/index/TestRollingUpdates.java
Date Wed, 01 Feb 2012 17:01:55 GMT
Author: mikemccand
Date: Wed Feb  1 17:01:55 2012
New Revision: 1239205

URL: http://svn.apache.org/viewvc?rev=1239205&view=rev
Log:
add doPackFST option (default to false) to MemoryCodec; randomize it during tests

Modified:
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java
    lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/index/RandomCodec.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java?rev=1239205&r1=1239204&r2=1239205&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java
(original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java
Wed Feb  1 17:01:55 2012
@@ -81,9 +81,21 @@ import org.apache.lucene.util.fst.Util;
 // the reality that it is actually written to disk, but
 // loads itself in ram?
 public class MemoryPostingsFormat extends PostingsFormat {
-  
+
+  private final boolean doPackFST;
+
   public MemoryPostingsFormat() {
+    this(false);
+  }
+
+  public MemoryPostingsFormat(boolean doPackFST) {
     super("Memory");
+    this.doPackFST = doPackFST;
+  }
+  
+  @Override
+  public String toString() {
+    return "PostingsFormat(name=" + getName() + " doPackFST= " + doPackFST + ")";
   }
 
   private static final boolean VERBOSE = false;
@@ -93,12 +105,14 @@ public class MemoryPostingsFormat extend
     private final FieldInfo field;
     private final Builder<BytesRef> builder;
     private final ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton();
+    private final boolean doPackFST;
     private int termCount;
 
-    public TermsWriter(IndexOutput out, FieldInfo field) {
+    public TermsWriter(IndexOutput out, FieldInfo field, boolean doPackFST) {
       this.out = out;
       this.field = field;
-      builder = new Builder<BytesRef>(FST.INPUT_TYPE.BYTE1, outputs);
+      this.doPackFST = doPackFST;
+      builder = new Builder<BytesRef>(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, Integer.MAX_VALUE,
outputs, null, doPackFST);
     }
 
     private class PostingsWriter extends PostingsConsumer {
@@ -230,7 +244,11 @@ public class MemoryPostingsFormat extend
         }
         out.writeVLong(sumDocFreq);
         out.writeVInt(docCount);
-        builder.finish().save(out);
+        FST<BytesRef> fst = builder.finish();
+        if (doPackFST) {
+          fst = fst.pack(3, Math.max(10, fst.getNodeCount()/4));
+        }
+        fst.save(out);
         if (VERBOSE) System.out.println("finish field=" + field.name + " fp=" + out.getFilePointer());
       }
     }
@@ -256,7 +274,7 @@ public class MemoryPostingsFormat extend
           throw new UnsupportedOperationException("this codec cannot index offsets");
         }
         if (VERBOSE) System.out.println("\naddField field=" + field.name);
-        return new TermsWriter(out, field);
+        return new TermsWriter(out, field, doPackFST);
       }
 
       @Override
@@ -776,6 +794,9 @@ public class MemoryPostingsFormat extend
           break;
         }
         final TermsReader termsReader = new TermsReader(state.fieldInfos, in, termCount);
+        if (VERBOSE) {
+          System.out.println("load field=" + termsReader.field.name);
+        }
         fields.put(termsReader.field.name, termsReader);
       }
     } finally {

Modified: lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/index/RandomCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/index/RandomCodec.java?rev=1239205&r1=1239204&r2=1239205&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/index/RandomCodec.java
(original)
+++ lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/index/RandomCodec.java
Wed Feb  1 17:01:55 2012
@@ -94,7 +94,7 @@ public class RandomCodec extends Lucene4
     formats.add(new Lucene40WithOrds());
     if (!useNoMemoryExpensiveCodec) {
       formats.add(new SimpleTextPostingsFormat());
-      formats.add(new MemoryPostingsFormat());
+      formats.add(new MemoryPostingsFormat(random.nextBoolean()));
     }
     Collections.shuffle(formats, random);
   }

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java?rev=1239205&r1=1239204&r2=1239205&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java Wed Feb
 1 17:01:55 2012
@@ -38,7 +38,7 @@ public class TestRollingUpdates extends 
 
     //provider.register(new MemoryCodec());
     if ( (!"Lucene3x".equals(Codec.getDefault().getName())) && random.nextBoolean())
{
-      Codec.setDefault(_TestUtil.alwaysPostingsFormat(new MemoryPostingsFormat()));
+      Codec.setDefault(_TestUtil.alwaysPostingsFormat(new MemoryPostingsFormat(random.nextBoolean())));
     }
 
     final IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(random)));



Mime
View raw message