Return-Path: X-Original-To: apmail-lucene-commits-archive@www.apache.org Delivered-To: apmail-lucene-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 519F09911 for ; Wed, 1 Feb 2012 17:02:18 +0000 (UTC) Received: (qmail 89727 invoked by uid 500); 1 Feb 2012 17:02:17 -0000 Mailing-List: contact commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@lucene.apache.org Delivered-To: mailing list commits@lucene.apache.org Received: (qmail 89718 invoked by uid 99); 1 Feb 2012 17:02:17 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 01 Feb 2012 17:02:17 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 01 Feb 2012 17:02:16 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 249D8238889B; Wed, 1 Feb 2012 17:01:56 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1239205 - in /lucene/dev/trunk/lucene/src: java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java test-framework/java/org/apache/lucene/index/RandomCodec.java test/org/apache/lucene/index/TestRollingUpdates.java Date: Wed, 01 Feb 2012 17:01:55 -0000 To: commits@lucene.apache.org From: mikemccand@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120201170156.249D8238889B@eris.apache.org> Author: mikemccand Date: Wed Feb 1 17:01:55 2012 New Revision: 1239205 URL: http://svn.apache.org/viewvc?rev=1239205&view=rev Log: add doPackFST option (default to false) to MemoryCodec; randomize it during tests Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/index/RandomCodec.java lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java?rev=1239205&r1=1239204&r2=1239205&view=diff ============================================================================== --- lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java (original) +++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java Wed Feb 1 17:01:55 2012 @@ -81,9 +81,21 @@ import org.apache.lucene.util.fst.Util; // the reality that it is actually written to disk, but // loads itself in ram? public class MemoryPostingsFormat extends PostingsFormat { - + + private final boolean doPackFST; + public MemoryPostingsFormat() { + this(false); + } + + public MemoryPostingsFormat(boolean doPackFST) { super("Memory"); + this.doPackFST = doPackFST; + } + + @Override + public String toString() { + return "PostingsFormat(name=" + getName() + " doPackFST= " + doPackFST + ")"; } private static final boolean VERBOSE = false; @@ -93,12 +105,14 @@ public class MemoryPostingsFormat extend private final FieldInfo field; private final Builder builder; private final ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton(); + private final boolean doPackFST; private int termCount; - public TermsWriter(IndexOutput out, FieldInfo field) { + public TermsWriter(IndexOutput out, FieldInfo field, boolean doPackFST) { this.out = out; this.field = field; - builder = new Builder(FST.INPUT_TYPE.BYTE1, outputs); + this.doPackFST = doPackFST; + builder = new Builder(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, Integer.MAX_VALUE, outputs, null, doPackFST); } private class PostingsWriter extends PostingsConsumer { @@ -230,7 +244,11 @@ public class MemoryPostingsFormat extend } out.writeVLong(sumDocFreq); out.writeVInt(docCount); - builder.finish().save(out); + FST fst = builder.finish(); + if (doPackFST) { + fst = fst.pack(3, Math.max(10, fst.getNodeCount()/4)); + } + fst.save(out); if (VERBOSE) System.out.println("finish field=" + field.name + " fp=" + out.getFilePointer()); } } @@ -256,7 +274,7 @@ public class MemoryPostingsFormat extend throw new UnsupportedOperationException("this codec cannot index offsets"); } if (VERBOSE) System.out.println("\naddField field=" + field.name); - return new TermsWriter(out, field); + return new TermsWriter(out, field, doPackFST); } @Override @@ -776,6 +794,9 @@ public class MemoryPostingsFormat extend break; } final TermsReader termsReader = new TermsReader(state.fieldInfos, in, termCount); + if (VERBOSE) { + System.out.println("load field=" + termsReader.field.name); + } fields.put(termsReader.field.name, termsReader); } } finally { Modified: lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/index/RandomCodec.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/index/RandomCodec.java?rev=1239205&r1=1239204&r2=1239205&view=diff ============================================================================== --- lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/index/RandomCodec.java (original) +++ lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/index/RandomCodec.java Wed Feb 1 17:01:55 2012 @@ -94,7 +94,7 @@ public class RandomCodec extends Lucene4 formats.add(new Lucene40WithOrds()); if (!useNoMemoryExpensiveCodec) { formats.add(new SimpleTextPostingsFormat()); - formats.add(new MemoryPostingsFormat()); + formats.add(new MemoryPostingsFormat(random.nextBoolean())); } Collections.shuffle(formats, random); } Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java?rev=1239205&r1=1239204&r2=1239205&view=diff ============================================================================== --- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java (original) +++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java Wed Feb 1 17:01:55 2012 @@ -38,7 +38,7 @@ public class TestRollingUpdates extends //provider.register(new MemoryCodec()); if ( (!"Lucene3x".equals(Codec.getDefault().getName())) && random.nextBoolean()) { - Codec.setDefault(_TestUtil.alwaysPostingsFormat(new MemoryPostingsFormat())); + Codec.setDefault(_TestUtil.alwaysPostingsFormat(new MemoryPostingsFormat(random.nextBoolean()))); } final IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));