Return-Path: 
 <commits-return-18798-apmail-lucene-commits-archive=www.apache.org@lucene.apache.org>
X-Original-To: apmail-lucene-commits-archive@www.apache.org
Delivered-To: apmail-lucene-commits-archive@www.apache.org
Received: from mail.apache.org (hermes.apache.org [140.211.11.3])
	by minotaur.apache.org (Postfix) with SMTP id 4A5D892DE
	for <apmail-lucene-commits-archive@www.apache.org>;
 Fri, 20 Jul 2012 01:02:51 +0000 (UTC)
Received: (qmail 28877 invoked by uid 500); 20 Jul 2012 01:02:51 -0000
Mailing-List: contact commits-help@lucene.apache.org; run by ezmlm
Precedence: bulk
List-Help: <mailto:commits-help@lucene.apache.org>
List-Unsubscribe: <mailto:commits-unsubscribe@lucene.apache.org>
List-Post: <mailto:commits@lucene.apache.org>
List-Id: <commits.lucene.apache.org>
Reply-To: dev@lucene.apache.org
Delivered-To: mailing list commits@lucene.apache.org
Received: (qmail 28783 invoked by uid 99); 20 Jul 2012 01:02:51 -0000
Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230)
    by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 20 Jul 2012 01:02:51 +0000
X-ASF-Spam-Status: No, hits=-2000.0 required=5.0
	tests=ALL_TRUSTED
X-Spam-Check-By: apache.org
Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4)
    by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 20 Jul 2012 01:02:32 +0000
Received: from eris.apache.org (localhost [127.0.0.1])
	by eris.apache.org (Postfix) with ESMTP id 8B67223889DA;
	Fri, 20 Jul 2012 01:02:10 +0000 (UTC)
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1363608 [4/10] - in /lucene/dev/branches/LUCENE-2878: ./
 dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/copyright/
 dev-tools/idea/.idea/libraries/ dev-tools/idea/lucene/ dev-tools/maven/
 dev-tools/maven/lucene/benchmark/ dev-tools/mave...
Date: Fri, 20 Jul 2012 01:01:52 -0000
To: commits@lucene.apache.org
From: simonw@apache.org
X-Mailer: svnmailer-1.0.8-patched
Message-Id: <20120720010210.8B67223889DA@eris.apache.org>

Modified: lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecContentSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecContentSource.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecContentSource.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecContentSource.java Fri Jul 20 01:01:39 2012
@@ -22,7 +22,6 @@ import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
-import java.io.Reader;
 import java.text.DateFormat;
 import java.text.ParsePosition;
 import java.text.SimpleDateFormat;
@@ -33,8 +32,6 @@ import java.util.Locale;
 import org.apache.lucene.benchmark.byTask.feeds.TrecDocParser.ParsePathType;
 import org.apache.lucene.benchmark.byTask.utils.Config;
 import org.apache.lucene.benchmark.byTask.utils.StreamUtils;
-import org.apache.lucene.benchmark.byTask.utils.StringBuilderReader;
-import org.apache.lucene.util.ThreadInterruptedException;
 
 /**
  * Implements a {@link ContentSource} over the TREC collection.
@@ -57,7 +54,7 @@ import org.apache.lucene.util.ThreadInte
  */
 public class TrecContentSource extends ContentSource {
 
-  private static final class DateFormatInfo {
+  static final class DateFormatInfo {
     DateFormat[] dfs;
     ParsePosition pos;
   }
@@ -83,13 +80,10 @@ public class TrecContentSource extends C
   };
 
   private ThreadLocal<DateFormatInfo> dateFormats = new ThreadLocal<DateFormatInfo>();
-  private ThreadLocal<StringBuilderReader> trecDocReader = new ThreadLocal<StringBuilderReader>();
   private ThreadLocal<StringBuilder> trecDocBuffer = new ThreadLocal<StringBuilder>();
   private File dataDir = null;
   private ArrayList<File> inputFiles = new ArrayList<File>();
   private int nextFile = 0;
-  private int rawDocSize = 0;
-
   // Use to synchronize threads on reading from the TREC documents.
   private Object lock = new Object();
 
@@ -126,17 +120,6 @@ public class TrecContentSource extends C
     return sb;
   }
   
-  Reader getTrecDocReader(StringBuilder docBuffer) {
-    StringBuilderReader r = trecDocReader.get();
-    if (r == null) {
-      r = new StringBuilderReader(docBuffer);
-      trecDocReader.set(r);
-    } else {
-      r.set(docBuffer);
-    }
-    return r;
-  }
-
   HTMLParser getHtmlParser() {
     return htmlParser;
   }
@@ -161,7 +144,7 @@ public class TrecContentSource extends C
         continue;
       }
 
-      rawDocSize += line.length();
+      line.length();
 
       if (lineStart!=null && line.startsWith(lineStart)) {
         if (collectMatchLine) {
@@ -287,12 +270,8 @@ public class TrecContentSource extends C
 
     // This code segment relies on HtmlParser being thread safe. When we get 
     // here, everything else is already private to that thread, so we're safe.
-    try {
-      docData = trecDocParser.parse(docData, name, this, docBuf, parsedPathType);
-      addItem();
-    } catch (InterruptedException ie) {
-      throw new ThreadInterruptedException(ie);
-    }
+    docData = trecDocParser.parse(docData, name, this, docBuf, parsedPathType);
+    addItem();
 
     return docData;
   }

Modified: lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecDocParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecDocParser.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecDocParser.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecDocParser.java Fri Jul 20 01:01:39 2012
@@ -80,7 +80,7 @@ public abstract class TrecDocParser {
    * parsers to alter their behavior according to the file path type. 
    */  
   public abstract DocData parse(DocData docData, String name, TrecContentSource trecSrc, 
-      StringBuilder docBuf, ParsePathType pathType) throws IOException, InterruptedException;
+      StringBuilder docBuf, ParsePathType pathType) throws IOException;
   
   /** 
    * strip tags from <code>buf</code>: each tag is replaced by a single blank.

Modified: lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFBISParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFBISParser.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFBISParser.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFBISParser.java Fri Jul 20 01:01:39 2012
@@ -37,7 +37,7 @@ public class TrecFBISParser extends Trec
 
   @Override
   public DocData parse(DocData docData, String name, TrecContentSource trecSrc, 
-      StringBuilder docBuf, ParsePathType pathType) throws IOException, InterruptedException {
+      StringBuilder docBuf, ParsePathType pathType) throws IOException {
     int mark = 0; // that much is skipped
     // optionally skip some of the text, set date, title
     Date date = null;

Modified: lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFR94Parser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFR94Parser.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFR94Parser.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFR94Parser.java Fri Jul 20 01:01:39 2012
@@ -41,7 +41,7 @@ public class TrecFR94Parser extends Trec
   
   @Override
   public DocData parse(DocData docData, String name, TrecContentSource trecSrc, 
-      StringBuilder docBuf, ParsePathType pathType) throws IOException, InterruptedException {
+      StringBuilder docBuf, ParsePathType pathType) throws IOException {
     int mark = 0; // that much is skipped
     // optionally skip some of the text, set date (no title?)
     Date date = null;

Modified: lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFTParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFTParser.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFTParser.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFTParser.java Fri Jul 20 01:01:39 2012
@@ -33,7 +33,7 @@ public class TrecFTParser extends TrecDo
 
   @Override
   public DocData parse(DocData docData, String name, TrecContentSource trecSrc, 
-      StringBuilder docBuf, ParsePathType pathType) throws IOException, InterruptedException {
+      StringBuilder docBuf, ParsePathType pathType) throws IOException {
     int mark = 0; // that much is skipped
 
     // date...

Modified: lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecGov2Parser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecGov2Parser.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecGov2Parser.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecGov2Parser.java Fri Jul 20 01:01:39 2012
@@ -18,7 +18,7 @@ package org.apache.lucene.benchmark.byTa
  */
 
 import java.io.IOException;
-import java.io.Reader;
+import java.io.StringReader;
 import java.util.Date;
 
 /**
@@ -31,29 +31,24 @@ public class TrecGov2Parser extends Trec
   
   private static final String DOCHDR = "<DOCHDR>";
   private static final String TERMINATING_DOCHDR = "</DOCHDR>";
-  private static final int TERMINATING_DOCHDR_LENGTH = TERMINATING_DOCHDR.length();
 
   @Override
   public DocData parse(DocData docData, String name, TrecContentSource trecSrc, 
-      StringBuilder docBuf, ParsePathType pathType) throws IOException, InterruptedException {
-    // Set up a (per-thread) reused Reader over the read content, reset it to re-read from docBuf
-    Reader r = trecSrc.getTrecDocReader(docBuf);
-
-    // skip some of the text, optionally set date
+      StringBuilder docBuf, ParsePathType pathType) throws IOException {
+    // skip some of the non-html text, optionally set date
     Date date = null;
-    int h1 = docBuf.indexOf(DOCHDR);
-    if (h1>=0) {
-      int h2 = docBuf.indexOf(TERMINATING_DOCHDR,h1);
-      String dateStr = extract(docBuf, DATE, DATE_END, h2, null);
+    int start = 0;
+    final int h1 = docBuf.indexOf(DOCHDR);
+    if (h1 >= 0) {
+      final int h2 = docBuf.indexOf(TERMINATING_DOCHDR, h1);
+      final String dateStr = extract(docBuf, DATE, DATE_END, h2, null);
       if (dateStr != null) {
         date = trecSrc.parseDate(dateStr);
       }
-      r.mark(h2+TERMINATING_DOCHDR_LENGTH);
+      start = h2 + TERMINATING_DOCHDR.length();
     }
-
-    r.reset();
-    HTMLParser htmlParser = trecSrc.getHtmlParser();
-    return htmlParser.parse(docData, name, date, null, r, null);
+    final String html = docBuf.substring(start);
+    return trecSrc.getHtmlParser().parse(docData, name, date, new StringReader(html), trecSrc);
   }
   
 }

Modified: lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecLATimesParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecLATimesParser.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecLATimesParser.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecLATimesParser.java Fri Jul 20 01:01:39 2012
@@ -36,7 +36,7 @@ public class TrecLATimesParser extends T
   
   @Override
   public DocData parse(DocData docData, String name, TrecContentSource trecSrc, 
-      StringBuilder docBuf, ParsePathType pathType) throws IOException, InterruptedException {
+      StringBuilder docBuf, ParsePathType pathType) throws IOException {
     int mark = 0; // that much is skipped
 
     // date...

Modified: lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecParserByPath.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecParserByPath.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecParserByPath.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecParserByPath.java Fri Jul 20 01:01:39 2012
@@ -26,7 +26,7 @@ public class TrecParserByPath extends Tr
 
   @Override
   public DocData parse(DocData docData, String name, TrecContentSource trecSrc, 
-      StringBuilder docBuf, ParsePathType pathType) throws IOException, InterruptedException {
+      StringBuilder docBuf, ParsePathType pathType) throws IOException {
     return pathType2parser.get(pathType).parse(docData, name, trecSrc, docBuf, pathType);
   }
 

Modified: lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/TrecContentSourceTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/TrecContentSourceTest.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/TrecContentSourceTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/TrecContentSourceTest.java Fri Jul 20 01:01:39 2012
@@ -166,6 +166,7 @@ public class TrecContentSourceTest exten
                   "<title>\r\n" + 
                   "TEST-001 title\r\n" + 
                   "</title>\r\n" + 
+                  "<meta name=\"date\" content=\"Tue&#44; 09 Dec 2003 22&#58;39&#58;08 GMT\">" +
                   "</head>\r\n" + 
                   "\r\n" + 
                   "<body>\r\n" + 
@@ -183,7 +184,7 @@ public class TrecContentSourceTest exten
     
     dd = source.getNextDocData(dd);
     assertDocData(dd, "TEST-001_0", "TEST-001 title", "TEST-001 text", source
-        .parseDate("Sun, 11 Jan 2009 08:01:00 GMT"));
+        .parseDate("Tue, 09 Dec 2003 22:39:08 GMT"));
     
     assertNoMoreDataException(source);
   }
@@ -331,6 +332,7 @@ public class TrecContentSourceTest exten
     dd = source.getNextDocData(dd);
     assertDocData(dd, "TEST-000_1", "TEST-000 title", "TEST-000 text", source
         .parseDate("Sun, 11 Jan 2009 08:00:00 GMT"));
+    source.close();
 
     // Don't test that NoMoreDataException is thrown, since the forever flag is
     // turned on.

Modified: lucene/dev/branches/LUCENE-2878/lucene/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/build.xml?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/build.xml (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/build.xml Fri Jul 20 01:01:39 2012
@@ -143,13 +143,9 @@
   <!--
    Run after Junit tests.
    -->
-  <target name="generate-clover-reports" depends="clover.check, clover">
+  <target name="generate-clover-reports" depends="clover">
+    <fail unless="run.clover">Clover not enabled!</fail>
     <mkdir dir="${clover.report.dir}"/>
-    <fileset dir="." id="clover.test.src.files">
-      <include name="**/src/test/**/*.java"/>
-      <include name="test-framework/src/java/**/*.java"/>
-      <exclude name="lucene/backwards/**"/>
-    </fileset>
     <fileset dir="build" id="clover.test.result.files">
       <include name="**/test/TEST-*.xml"/>
       <!-- do not include BW tests -->
@@ -158,25 +154,25 @@
     <clover-report>
       <current outfile="${clover.report.dir}" title="${final.name}" numThreads="0">
         <format type="html" filter="assert"/>
-        <testsources refid="clover.test.src.files"/>
         <testresults refid="clover.test.result.files"/>
       </current>
       <current outfile="${clover.report.dir}/clover.xml" title="${final.name}">
         <format type="xml" filter="assert"/>
-        <testsources refid="clover.test.src.files"/>
         <testresults refid="clover.test.result.files"/>
       </current>
     </clover-report>
   </target>
 
   <!-- Validation (license/notice/api checks). -->
-  <target name="validate" depends="check-licenses,check-forbidden-apis" description="Validate stuff." />
+  <target name="validate" depends="check-licenses,rat-sources,check-forbidden-apis" description="Validate stuff." />
 
   <target name="check-licenses" depends="compile-tools,resolve,load-custom-tasks" description="Validate license stuff.">
     <license-check-macro dir="${basedir}" />
   </target>
 
-  <target name="check-forbidden-apis" depends="compile-tools,compile-test,load-custom-tasks" description="Check forbidden API calls in compiled class files.">
+  <target name="check-forbidden-apis" depends="compile-tools,compile-test,load-custom-tasks,-check-forbidden-jdk-apis,-check-forbidden-test-apis,-check-system-out" description="Check forbidden API calls in compiled class files"/>
+
+  <target name="-check-forbidden-jdk-apis">
     <forbidden-apis> 
       <apiFileSet dir="${custom-tasks.dir}/forbiddenApis">
         <include name="jdk.txt" />
@@ -186,11 +182,53 @@
     </forbidden-apis>
   </target>
 
+  <target name="-check-forbidden-test-apis">
+    <forbidden-apis apiFile="${custom-tasks.dir}/forbiddenApis/tests.txt"> 
+      <classpath refid="junit-path"/>
+      <fileset dir="${basedir}/build" includes="**/classes/test/**/*.class,test-framework/**/*.class" />
+    </forbidden-apis>
+  </target>
+
+  <target name="-check-system-out">
+    <forbidden-apis apiFile="${custom-tasks.dir}/forbiddenApis/system-out.txt">
+      <fileset dir="${basedir}/build">
+        <include name="**/classes/java/**/*.class"/>
+        <!-- this is basically tests -->
+        <exclude name="test-framework/**"/>
+        <!-- exclude command line tools -->
+        <exclude name="core/classes/java/org/apache/lucene/index/CheckIndex.class"/>
+        <exclude name="core/classes/java/org/apache/lucene/index/IndexUpgrader.class"/>
+        <exclude name="core/classes/java/org/apache/lucene/store/LockVerifyServer.class"/>
+        <exclude name="core/classes/java/org/apache/lucene/store/LockStressTest.class"/>
+        <exclude name="analysis/stempel/classes/java/org/egothor/stemmer/Compile.class"/>
+        <exclude name="analysis/stempel/classes/java/org/egothor/stemmer/DiffIt.class"/>
+        <exclude name="benchmark/**"/>
+        <exclude name="demo/classes/java/org/apache/lucene/demo/IndexFiles.class"/>
+        <exclude name="demo/classes/java/org/apache/lucene/demo/SearchFiles.class"/>
+        <exclude name="misc/classes/java/org/apache/lucene/index/CompoundFileExtractor.class"/>
+        <exclude name="misc/classes/java/org/apache/lucene/index/IndexSplitter.class"/>
+        <exclude name="misc/classes/java/org/apache/lucene/index/MultiPassIndexSplitter.class"/>
+        <exclude name="misc/classes/java/org/apache/lucene/misc/GetTermInfo.class"/>
+        <exclude name="misc/classes/java/org/apache/lucene/misc/HighFreqTerms.class"/>
+        <exclude name="misc/classes/java/org/apache/lucene/misc/IndexMergeTool.class"/>
+        <!-- not printing, just checking if its a system stream -->
+        <exclude name="core/classes/java/org/apache/lucene/util/PrintStreamInfoStream.class"/>
+        <!-- not printing, just generated code from javacc -->
+        <exclude name="queryparser/classes/java/org/apache/lucene/queryparser/classic/QueryParserTokenManager.class"/>
+        <exclude name="queryparser/classes/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParserTokenManager.class"/>
+        <exclude name="queryparser/classes/java/org/apache/lucene/queryparser/surround/parser/QueryParserTokenManager.class"/>
+      </fileset>
+    </forbidden-apis>
+  </target>
+
   <target name="resolve">
     <sequential>
       <ant dir="test-framework" target="resolve" inheritall="false">
          <propertyset refid="uptodate.and.compiled.properties"/>
       </ant>
+      <ant dir="${common.dir}/tools" target="resolve" inheritAll="false">
+         <propertyset refid="uptodate.and.compiled.properties"/>
+      </ant>
       <modules-crawl target="resolve" failonerror="true"/>
     </sequential>
   </target>
@@ -374,6 +412,8 @@
 
     <!-- Exclude javadoc package-list files under licenses incompatible with the ASL -->
     <delete dir="${svn.export.dir}/tools/javadoc/java6"/>
+    <!-- Exclude clover license files incompatible with the ASL -->
+    <delete dir="${svn.export.dir}/tools/clover"/>
 
     <build-changes changes.src.dir="${svn.export.dir}/site/changes"
                    changes.target.dir="${svn.export.dir}/docs/changes"/>

Modified: lucene/dev/branches/LUCENE-2878/lucene/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/common-build.xml?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/common-build.xml (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/common-build.xml Fri Jul 20 01:01:39 2012
@@ -95,7 +95,17 @@
   <property name="tests.asserts.gracious" value="false"/>
   <property name="tests.verbose" value="false"/>
   <property name="tests.infostream" value="${tests.verbose}"/>
+  
+  <condition property="tests.heapsize" value="768M">
+    <isset property="run.clover"/>
+  </condition>
   <property name="tests.heapsize" value="512M"/>
+  
+  <condition property="tests.clover.args" value="-XX:ReservedCodeCacheSize=128m">
+    <isset property="run.clover"/>
+  </condition>
+  <property name="tests.clover.args" value=""/>
+  
   <property name="tests.tempDir" value="${build.dir}/test"/>
 
   <property name="tests.cachefile" location="${common.dir}/tools/junit4/cached-timehints.txt" />
@@ -124,9 +134,6 @@
   <property name="javac.target" value="1.6"/>
   <property name="javac.source.backwards" value="1.6"/>
   <property name="javac.target.backwards" value="1.6"/>
-  <!-- clover wants to run with -lib, otherwise we prefer a repeatable
-       classpath -->
-  <property name="javac.includeAntRuntime" value="${run.clover}"/>
   <property name="javac.args" value="-Xlint -Xlint:-deprecation -Xlint:-serial"/>
   <property name="bootclasspath" value=""/>
   <property name="javadoc.link" value="http://download.oracle.com/javase/6/docs/api/"/>
@@ -155,7 +162,6 @@
 
   <property name="src.dir" location="src/java"/>
   <property name="tests.src.dir" location="src/test"/>
-  <property name="tests-framework.src.dir" location="${common.dir}/test-framework/src/java"/>
   <property name="build.dir" location="build"/>
   <!-- Needed in case a module needs the original build, also for compile-tools to be called from a module -->
   <property name="common.build.dir" location="${common.dir}/build"/>
@@ -213,31 +219,20 @@
 
   <property name="filtered.pom.templates.dir" location="${common.dir}/build/poms"/>
 
-  <property name="clover.db.dir" location="${common.dir}/build/test/clover/db"/>
-  <property name="clover.report.dir" location="${common.dir}/build/test/clover/reports"/>
+  <property name="clover.license.path" location="${common.dir}/tools/clover/clover.license"/>
+  <property name="clover.db.dir" location="${common.dir}/build/clover/db"/>
+  <property name="clover.report.dir" location="${common.dir}/build/clover/reports"/>
 
   <!-- a reasonable default exclusion set, can be overridden for special cases -->
   <property name="rat.excludes" value="**/TODO,**/*.txt"/>
 
-  <available
-            property="clover.present"
-            classname="com.cenqua.clover.tasks.CloverReportTask"
-  />
-   
-  <condition property="clover.enabled">
-    <and>
-     <isset property="run.clover"/>
-     <isset property="clover.present"/>
-    </and>
-  </condition>
-
   <propertyset id="uptodate.and.compiled.properties" dynamic="true">
     <propertyref regex=".*\.uptodate$$"/>
     <propertyref regex=".*\.compiled$$"/>
   </propertyset>
 
   <patternset id="lucene.local.src.package.patterns"
-              excludes="**/pom.xml,**/*.iml,**/*.jar,build/**,dist/**,lucene/benchmark/work/**,lucene/benchmark/temp/**"
+              excludes="**/pom.xml,**/*.iml,**/*.jar,build/**,dist/**,benchmark/work/**,benchmark/temp/**,tools/javadoc/java6/**,tools/clover/**"
   />
 
   <!-- Default exclude sources and javadoc jars from Ivy fetch to save time and bandwidth -->
@@ -793,6 +788,7 @@
         >
             <!-- Classpaths. -->
             <classpath refid="@{junit.classpath}"/>
+            <classpath refid="clover.classpath" />
 
             <!-- Assertions. -->
             <assertions>
@@ -802,6 +798,7 @@
 
             <!-- JVM arguments and system properties. -->
             <jvmarg line="${args}"/>
+            <jvmarg line="${tests.clover.args}"/>
 
             <!-- set the number of times tests should run -->
             <sysproperty key="tests.iters" value="${tests.iters}"/>
@@ -1091,48 +1088,55 @@ ${tests-output}/junit4-*.suites     - pe
 
   <!-- note: order here is important, the taskdef depends on test-framework
        jars so we just order it after compile-test to ensure that -->
-  <target name="test" depends="compile-test,install-junit4-taskdef,validate" description="Runs unit tests">
+  <target name="test" depends="compile-test,install-junit4-taskdef,clover,validate" description="Runs unit tests">
     <mkdir dir="${junit.output.dir}"/>
     <test-macro threadNum="${tests.jvms}" />
   </target>
 
   <!--
-   If you want clover test code coverage, run this before the tests.  You need clover.jar and the license in your ANT classspath and you need to specify -Drun.clover=true on the command line.
-
    See http://issues.apache.org/jira/browse/LUCENE-721
    -->
-  <target name="clover" depends="clover.setup, clover.info" description="Instrument the Unit tests using Clover.  Requires a Clover 2.x license and clover.jar in the ANT classpath.  To use, specify -Drun.clover=true on the command line."/>
+  <target name="clover" depends="-clover.disable,-clover.setup" description="Instrument the Unit tests using Clover. To use, specify -Drun.clover=true on the command line."/>
+  
+  <target name="-clover.setup" if="run.clover" unless="clover.loaded">
+    <available file="${clover.license.path}" property="clover.license.available" />
+    <fail unless="clover.license.available"><![CDATA[.
 
-  <target name="clover.setup" if="clover.enabled">
-    <taskdef resource="cloverlib.xml"/>
+  #########################################################################
+  Atlassian Clover License not found!
+  
+  Current License path: ${clover.license.path}
+
+  To use Atlassian Clover with Lucene build, you need a proper license
+  and let the system property 'clover.license.path' point to it.
+  You can pass it to ANT with:
+  
+  $ ant -Dclover.license.path=/path/to/clover.license -Drun.clover=true ...
+  
+  Apache Lucene/Solr source checkouts from SVN already contain the
+  file, but source distributions cannot because of legal reasons.
+  #########################################################################
+  
+]]></fail>
+    <echo>Code coverage with Atlassian Clover enabled.</echo>
+    <ivy:cachepath organisation="com.cenqua.clover" module="clover" revision="2.6.3"
+      inline="true" conf="master" type="jar" pathid="clover.classpath"/>
+    <taskdef resource="cloverlib.xml" classpathref="clover.classpath" />
     <mkdir dir="${clover.db.dir}"/>
-    <clover-setup initString="${clover.db.dir}/lucene_coverage.db" encoding="${build.encoding}">
-      <fileset dir="${src.dir}">
+    <clover-setup initString="${clover.db.dir}/coverage.db" encoding="${build.encoding}">
+      <fileset dir="${src.dir}" erroronmissingdir="no">
         <include name="org/apache/**/*.java" />
       </fileset>
-      <testsources dir="${tests-framework.src.dir}">
-        <include name="org/apache/**/*.java" />
-      </testsources>
-      <testsources dir="${tests.src.dir}">
+      <testsources dir="${tests.src.dir}" erroronmissingdir="no">
         <include name="org/apache/**/*.java" />
       </testsources>
     </clover-setup>
+    <property name="clover.loaded" value="true"/>
   </target>
 
-  <target name="clover.info" unless="clover.present">
-  	<echo>
-      Clover not found. Code coverage reports disabled.
-  	</echo>
-  </target>
-
-  <target name="clover.check">
-	<fail unless="clover.present">
-	  ##################################################################
-      Clover not found.
-      Please make sure clover.jar is in ANT_HOME/lib, or made available
-      to Ant using other mechanisms like -lib or CLASSPATH.
-      ##################################################################
-  	</fail>
+  <target name="-clover.disable" unless="run.clover">
+    <!-- define dummy clover path used by junit -->
+    <path id="clover.classpath"/>
   </target>
 
   <target name="generate-test-reports" description="Generates test reports">
@@ -1259,13 +1263,10 @@ ${tests-output}/junit4-*.suites     - pe
     </ant>
   </target>
 
-  <!-- TODO: add cool detector like the ivy one? this requires you have rat -->
-  <target name="rat-sources-typedef">
-    <typedef resource="org/apache/rat/anttasks/antlib.xml" uri="antlib:rat.anttasks">
-      <classpath>
-        <fileset dir="." includes="rat*.jar"/>
-      </classpath>
-    </typedef>
+  <target name="rat-sources-typedef" unless="rat.loaded">
+    <ivy:cachepath organisation="org.apache.rat" module="apache-rat" revision="0.8" transitive="false" inline="true" conf="master" type="jar" pathid="rat.classpath"/>
+    <typedef resource="org/apache/rat/anttasks/antlib.xml" uri="antlib:org.apache.rat.anttasks" classpathref="rat.classpath"/>
+    <property name="rat.loaded" value="true"/>
   </target>
 
   <target name="rat-sources" depends="rat-sources-typedef"

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java Fri Jul 20 01:01:39 2012
@@ -69,7 +69,7 @@ public abstract class Analyzer {
    * instance of {@link TokenStreamComponents}. It returns the sink of the
    * components and stores the components internally. Subsequent calls to this
    * method will reuse the previously stored components after resetting them
-   * through {@link TokenStreamComponents#reset(Reader)}.
+   * through {@link TokenStreamComponents#setReader(Reader)}.
    * </p>
    * 
    * @param fieldName the name of the field the created TokenStream is used for
@@ -83,7 +83,7 @@ public abstract class Analyzer {
       components = createComponents(fieldName, r);
       reuseStrategy.setReusableComponents(fieldName, components);
     } else {
-      components.reset(r);
+      components.setReader(r);
     }
     return components.getTokenStream();
   }
@@ -181,8 +181,8 @@ public abstract class Analyzer {
      * @throws IOException
      *           if the component's reset method throws an {@link IOException}
      */
-    protected void reset(final Reader reader) throws IOException {
-      source.reset(reader);
+    protected void setReader(final Reader reader) throws IOException {
+      source.setReader(reader);
     }
 
     /**

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java Fri Jul 20 01:01:39 2012
@@ -65,21 +65,21 @@ public abstract class Tokenizer extends 
     }
   }
   
-  /** Return the corrected offset. If {@link #input} is a {@link CharStream} subclass
-   * this method calls {@link CharStream#correctOffset}, else returns <code>currentOff</code>.
+  /** Return the corrected offset. If {@link #input} is a {@link CharFilter} subclass
+   * this method calls {@link CharFilter#correctOffset}, else returns <code>currentOff</code>.
    * @param currentOff offset as seen in the output
    * @return corrected offset based on the input
-   * @see CharStream#correctOffset
+   * @see CharFilter#correctOffset
    */
   protected final int correctOffset(int currentOff) {
     assert input != null: "this tokenizer is closed";
-    return (input instanceof CharStream) ? ((CharStream) input).correctOffset(currentOff) : currentOff;
+    return (input instanceof CharFilter) ? ((CharFilter) input).correctOffset(currentOff) : currentOff;
   }
 
   /** Expert: Reset the tokenizer to a new reader.  Typically, an
    *  analyzer (in its tokenStream method) will use
    *  this to re-use a previously created tokenizer. */
-  public void reset(Reader input) throws IOException {
+  public void setReader(Reader input) throws IOException {
     assert input != null: "input must not be null";
     this.input = input;
   }

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java Fri Jul 20 01:01:39 2012
@@ -1958,11 +1958,11 @@ public class BlockTreeTermsReader extend
       }
 
       @SuppressWarnings("unused")
-      private void printSeekState() throws IOException {
+      private void printSeekState(PrintStream out) throws IOException {
         if (currentFrame == staticFrame) {
-          System.out.println("  no prior seek");
+          out.println("  no prior seek");
         } else {
-          System.out.println("  prior seek state:");
+          out.println("  prior seek state:");
           int ord = 0;
           boolean isSeekFrame = true;
           while(true) {
@@ -1970,26 +1970,26 @@ public class BlockTreeTermsReader extend
             assert f != null;
             final BytesRef prefix = new BytesRef(term.bytes, 0, f.prefix);
             if (f.nextEnt == -1) {
-              System.out.println("    frame " + (isSeekFrame ? "(seek)" : "(next)") + " ord=" + ord + " fp=" + f.fp + (f.isFloor ? (" (fpOrig=" + f.fpOrig + ")") : "") + " prefixLen=" + f.prefix + " prefix=" + prefix + (f.nextEnt == -1 ? "" : (" (of " + f.entCount + ")")) + " hasTerms=" + f.hasTerms + " isFloor=" + f.isFloor + " code=" + ((f.fp<<BlockTreeTermsWriter.OUTPUT_FLAGS_NUM_BITS) + (f.hasTerms ? BlockTreeTermsWriter.OUTPUT_FLAG_HAS_TERMS:0) + (f.isFloor ? BlockTreeTermsWriter.OUTPUT_FLAG_IS_FLOOR:0)) + " isLastInFloor=" + f.isLastInFloor + " mdUpto=" + f.metaDataUpto + " tbOrd=" + f.getTermBlockOrd());
+              out.println("    frame " + (isSeekFrame ? "(seek)" : "(next)") + " ord=" + ord + " fp=" + f.fp + (f.isFloor ? (" (fpOrig=" + f.fpOrig + ")") : "") + " prefixLen=" + f.prefix + " prefix=" + prefix + (f.nextEnt == -1 ? "" : (" (of " + f.entCount + ")")) + " hasTerms=" + f.hasTerms + " isFloor=" + f.isFloor + " code=" + ((f.fp<<BlockTreeTermsWriter.OUTPUT_FLAGS_NUM_BITS) + (f.hasTerms ? BlockTreeTermsWriter.OUTPUT_FLAG_HAS_TERMS:0) + (f.isFloor ? BlockTreeTermsWriter.OUTPUT_FLAG_IS_FLOOR:0)) + " isLastInFloor=" + f.isLastInFloor + " mdUpto=" + f.metaDataUpto + " tbOrd=" + f.getTermBlockOrd());
             } else {
-              System.out.println("    frame " + (isSeekFrame ? "(seek, loaded)" : "(next, loaded)") + " ord=" + ord + " fp=" + f.fp + (f.isFloor ? (" (fpOrig=" + f.fpOrig + ")") : "") + " prefixLen=" + f.prefix + " prefix=" + prefix + " nextEnt=" + f.nextEnt + (f.nextEnt == -1 ? "" : (" (of " + f.entCount + ")")) + " hasTerms=" + f.hasTerms + " isFloor=" + f.isFloor + " code=" + ((f.fp<<BlockTreeTermsWriter.OUTPUT_FLAGS_NUM_BITS) + (f.hasTerms ? BlockTreeTermsWriter.OUTPUT_FLAG_HAS_TERMS:0) + (f.isFloor ? BlockTreeTermsWriter.OUTPUT_FLAG_IS_FLOOR:0)) + " lastSubFP=" + f.lastSubFP + " isLastInFloor=" + f.isLastInFloor + " mdUpto=" + f.metaDataUpto + " tbOrd=" + f.getTermBlockOrd());
+              out.println("    frame " + (isSeekFrame ? "(seek, loaded)" : "(next, loaded)") + " ord=" + ord + " fp=" + f.fp + (f.isFloor ? (" (fpOrig=" + f.fpOrig + ")") : "") + " prefixLen=" + f.prefix + " prefix=" + prefix + " nextEnt=" + f.nextEnt + (f.nextEnt == -1 ? "" : (" (of " + f.entCount + ")")) + " hasTerms=" + f.hasTerms + " isFloor=" + f.isFloor + " code=" + ((f.fp<<BlockTreeTermsWriter.OUTPUT_FLAGS_NUM_BITS) + (f.hasTerms ? BlockTreeTermsWriter.OUTPUT_FLAG_HAS_TERMS:0) + (f.isFloor ? BlockTreeTermsWriter.OUTPUT_FLAG_IS_FLOOR:0)) + " lastSubFP=" + f.lastSubFP + " isLastInFloor=" + f.isLastInFloor + " mdUpto=" + f.metaDataUpto + " tbOrd=" + f.getTermBlockOrd());
             }
             if (index != null) {
               assert !isSeekFrame || f.arc != null: "isSeekFrame=" + isSeekFrame + " f.arc=" + f.arc;
               if (f.prefix > 0 && isSeekFrame && f.arc.label != (term.bytes[f.prefix-1]&0xFF)) {
-                System.out.println("      broken seek state: arc.label=" + (char) f.arc.label + " vs term byte=" + (char) (term.bytes[f.prefix-1]&0xFF));
+                out.println("      broken seek state: arc.label=" + (char) f.arc.label + " vs term byte=" + (char) (term.bytes[f.prefix-1]&0xFF));
                 throw new RuntimeException("seek state is broken");
               }
               BytesRef output = Util.get(index, prefix);
               if (output == null) {
-                System.out.println("      broken seek state: prefix is not final in index");
+                out.println("      broken seek state: prefix is not final in index");
                 throw new RuntimeException("seek state is broken");
               } else if (isSeekFrame && !f.isFloor) {
                 final ByteArrayDataInput reader = new ByteArrayDataInput(output.bytes, output.offset, output.length);
                 final long codeOrig = reader.readVLong();
                 final long code = (f.fp << BlockTreeTermsWriter.OUTPUT_FLAGS_NUM_BITS) | (f.hasTerms ? BlockTreeTermsWriter.OUTPUT_FLAG_HAS_TERMS:0) | (f.isFloor ? BlockTreeTermsWriter.OUTPUT_FLAG_IS_FLOOR:0);
                 if (codeOrig != code) {
-                  System.out.println("      broken seek state: output code=" + codeOrig + " doesn't match frame code=" + code);
+                  out.println("      broken seek state: output code=" + codeOrig + " doesn't match frame code=" + code);
                   throw new RuntimeException("seek state is broken");
                 }
               }

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsWriter.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsWriter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsWriter.java Fri Jul 20 01:01:39 2012
@@ -113,7 +113,7 @@ public final class PulsingPostingsWriter
 
   @Override
   public void startTerm() {
-    if (DEBUG) System.out.println("PW   startTerm");
+    //if (DEBUG) System.out.println("PW   startTerm");
     assert pendingCount == 0;
   }
 
@@ -125,7 +125,7 @@ public final class PulsingPostingsWriter
   @Override
   public void setField(FieldInfo fieldInfo) {
     this.indexOptions = fieldInfo.getIndexOptions();
-    if (DEBUG) System.out.println("PW field=" + fieldInfo.name + " indexOptions=" + indexOptions);
+    //if (DEBUG) System.out.println("PW field=" + fieldInfo.name + " indexOptions=" + indexOptions);
     storePayloads = fieldInfo.hasPayloads();
     wrappedPostingsWriter.setField(fieldInfo);
     //DEBUG = BlockTreeTermsWriter.DEBUG;
@@ -147,11 +147,11 @@ public final class PulsingPostingsWriter
     }
     */
 
-    if (DEBUG) System.out.println("PW     doc=" + docID);
+    //if (DEBUG) System.out.println("PW     doc=" + docID);
 
     if (pendingCount == pending.length) {
       push();
-      if (DEBUG) System.out.println("PW: wrapped.finishDoc");
+      //if (DEBUG) System.out.println("PW: wrapped.finishDoc");
       wrappedPostingsWriter.finishDoc();
     }
 
@@ -177,7 +177,7 @@ public final class PulsingPostingsWriter
   @Override
   public void addPosition(int position, BytesRef payload, int startOffset, int endOffset) throws IOException {
 
-    if (DEBUG) System.out.println("PW       pos=" + position + " payload=" + (payload == null ? "null" : payload.length + " bytes"));
+    //if (DEBUG) System.out.println("PW       pos=" + position + " payload=" + (payload == null ? "null" : payload.length + " bytes"));
     if (pendingCount == pending.length) {
       push();
     }
@@ -207,7 +207,7 @@ public final class PulsingPostingsWriter
 
   @Override
   public void finishDoc() throws IOException {
-    if (DEBUG) System.out.println("PW     finishDoc");
+    // if (DEBUG) System.out.println("PW     finishDoc");
     if (pendingCount == -1) {
       wrappedPostingsWriter.finishDoc();
     }
@@ -220,7 +220,7 @@ public final class PulsingPostingsWriter
   /** Called when we are done adding docs to this term */
   @Override
   public void finishTerm(TermStats stats) throws IOException {
-    if (DEBUG) System.out.println("PW   finishTerm docCount=" + stats.docFreq + " pendingCount=" + pendingCount + " pendingTerms.size()=" + pendingTerms.size());
+    // if (DEBUG) System.out.println("PW   finishTerm docCount=" + stats.docFreq + " pendingCount=" + pendingCount + " pendingTerms.size()=" + pendingTerms.size());
 
     assert pendingCount > 0 || pendingCount == -1;
 
@@ -252,7 +252,7 @@ public final class PulsingPostingsWriter
           final int delta = doc.docID - lastDocID;
           lastDocID = doc.docID;
 
-          if (DEBUG) System.out.println("  write doc=" + doc.docID + " freq=" + doc.termFreq);
+          // if (DEBUG) System.out.println("  write doc=" + doc.docID + " freq=" + doc.termFreq);
 
           if (doc.termFreq == 1) {
             buffer.writeVInt((delta<<1)|1);
@@ -268,7 +268,7 @@ public final class PulsingPostingsWriter
             assert pos.docID == doc.docID;
             final int posDelta = pos.pos - lastPos;
             lastPos = pos.pos;
-            if (DEBUG) System.out.println("    write pos=" + pos.pos);
+            // if (DEBUG) System.out.println("    write pos=" + pos.pos);
             final int payloadLength = pos.payload == null ? 0 : pos.payload.length;
             if (storePayloads) {
               if (payloadLength != lastPayloadLength) {
@@ -341,7 +341,7 @@ public final class PulsingPostingsWriter
 
   @Override
   public void flushTermsBlock(int start, int count) throws IOException {
-    if (DEBUG) System.out.println("PW: flushTermsBlock start=" + start + " count=" + count + " pendingTerms.size()=" + pendingTerms.size());
+    // if (DEBUG) System.out.println("PW: flushTermsBlock start=" + start + " count=" + count + " pendingTerms.size()=" + pendingTerms.size());
     int wrappedCount = 0;
     assert buffer.getFilePointer() == 0;
     assert start >= count;
@@ -375,7 +375,7 @@ public final class PulsingPostingsWriter
     // Remove the terms we just wrote:
     pendingTerms.subList(pendingTerms.size()-start, limit).clear();
 
-    if (DEBUG) System.out.println("PW:   len=" + buffer.getFilePointer() + " fp=" + termsOut.getFilePointer() + " futureWrappedCount=" + futureWrappedCount + " wrappedCount=" + wrappedCount);
+    // if (DEBUG) System.out.println("PW:   len=" + buffer.getFilePointer() + " fp=" + termsOut.getFilePointer() + " futureWrappedCount=" + futureWrappedCount + " wrappedCount=" + wrappedCount);
     // TODO: can we avoid calling this if all terms
     // were inlined...?  Eg for a "primary key" field, the
     // wrapped codec is never invoked...
@@ -384,7 +384,7 @@ public final class PulsingPostingsWriter
 
   // Pushes pending positions to the wrapped codec
   private void push() throws IOException {
-    if (DEBUG) System.out.println("PW now push @ " + pendingCount + " wrapped=" + wrappedPostingsWriter);
+    // if (DEBUG) System.out.println("PW now push @ " + pendingCount + " wrapped=" + wrappedPostingsWriter);
     assert pendingCount == pending.length;
       
     wrappedPostingsWriter.startTerm();
@@ -395,17 +395,17 @@ public final class PulsingPostingsWriter
       for(Position pos : pending) {
         if (doc == null) {
           doc = pos;
-          if (DEBUG) System.out.println("PW: wrapped.startDoc docID=" + doc.docID + " tf=" + doc.termFreq);
+          // if (DEBUG) System.out.println("PW: wrapped.startDoc docID=" + doc.docID + " tf=" + doc.termFreq);
           wrappedPostingsWriter.startDoc(doc.docID, doc.termFreq);
         } else if (doc.docID != pos.docID) {
           assert pos.docID > doc.docID;
-          if (DEBUG) System.out.println("PW: wrapped.finishDoc");
+          // if (DEBUG) System.out.println("PW: wrapped.finishDoc");
           wrappedPostingsWriter.finishDoc();
           doc = pos;
-          if (DEBUG) System.out.println("PW: wrapped.startDoc docID=" + doc.docID + " tf=" + doc.termFreq);
+          // if (DEBUG) System.out.println("PW: wrapped.startDoc docID=" + doc.docID + " tf=" + doc.termFreq);
           wrappedPostingsWriter.startDoc(doc.docID, doc.termFreq);
         }
-        if (DEBUG) System.out.println("PW:   wrapped.addPos pos=" + pos.pos);
+        // if (DEBUG) System.out.println("PW:   wrapped.addPos pos=" + pos.pos);
         wrappedPostingsWriter.addPosition(pos.pos, pos.payload, pos.startOffset, pos.endOffset);
       }
       //wrappedPostingsWriter.finishDoc();

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java Fri Jul 20 01:01:39 2012
@@ -668,7 +668,7 @@ public class CheckIndex {
    * checks Fields api is consistent with itself.
    * searcher is optional, to verify with queries. Can be null.
    */
-  private Status.TermIndexStatus checkFields(Fields fields, Bits liveDocs, int maxDoc, FieldInfos fieldInfos, boolean doPrint) throws IOException {
+  private Status.TermIndexStatus checkFields(Fields fields, Bits liveDocs, int maxDoc, FieldInfos fieldInfos, boolean doPrint, boolean isVectors) throws IOException {
     // TODO: we should probably return our own stats thing...?!
     
     final Status.TermIndexStatus status = new Status.TermIndexStatus();
@@ -863,17 +863,21 @@ public class CheckIndex {
               if (hasOffsets) {
                 int startOffset = postings.startOffset();
                 int endOffset = postings.endOffset();
-                if (startOffset < 0) {
-                  throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + ": startOffset " + startOffset + " is out of bounds");
-                }
-                if (startOffset < lastOffset) {
-                  throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + ": startOffset " + startOffset + " < lastStartOffset " + lastOffset);
-                }
-                if (endOffset < 0) {
-                  throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + ": endOffset " + endOffset + " is out of bounds");
-                }
-                if (endOffset < startOffset) {
-                  throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + ": endOffset " + endOffset + " < startOffset " + startOffset);
+                // NOTE: we cannot enforce any bounds whatsoever on vectors... they were a free-for-all before?
+                // but for offsets in the postings lists these checks are fine: they were always enforced by IndexWriter
+                if (!isVectors) {
+                  if (startOffset < 0) {
+                    throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + ": startOffset " + startOffset + " is out of bounds");
+                  }
+                  if (startOffset < lastOffset) {
+                    throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + ": startOffset " + startOffset + " < lastStartOffset " + lastOffset);
+                  }
+                  if (endOffset < 0) {
+                    throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + ": endOffset " + endOffset + " is out of bounds");
+                  }
+                  if (endOffset < startOffset) {
+                    throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + ": endOffset " + endOffset + " < startOffset " + startOffset);
+                  }
                 }
                 lastOffset = startOffset;
               }
@@ -956,17 +960,21 @@ public class CheckIndex {
                 if (hasOffsets) {
                   int startOffset = postings.startOffset();
                   int endOffset = postings.endOffset();
-                  if (startOffset < 0) {
-                    throw new RuntimeException("term " + term + ": doc " + docID + ": pos " + pos + ": startOffset " + startOffset + " is out of bounds");
-                  }
-                  if (startOffset < lastOffset) {
-                    throw new RuntimeException("term " + term + ": doc " + docID + ": pos " + pos + ": startOffset " + startOffset + " < lastStartOffset " + lastOffset);
-                  }
-                  if (endOffset < 0) {
-                    throw new RuntimeException("term " + term + ": doc " + docID + ": pos " + pos + ": endOffset " + endOffset + " is out of bounds");
-                  }
-                  if (endOffset < startOffset) {
-                    throw new RuntimeException("term " + term + ": doc " + docID + ": pos " + pos + ": endOffset " + endOffset + " < startOffset " + startOffset);
+                  // NOTE: we cannot enforce any bounds whatsoever on vectors... they were a free-for-all before?
+                  // but for offsets in the postings lists these checks are fine: they were always enforced by IndexWriter
+                  if (!isVectors) {
+                    if (startOffset < 0) {
+                      throw new RuntimeException("term " + term + ": doc " + docID + ": pos " + pos + ": startOffset " + startOffset + " is out of bounds");
+                    }
+                    if (startOffset < lastOffset) {
+                      throw new RuntimeException("term " + term + ": doc " + docID + ": pos " + pos + ": startOffset " + startOffset + " < lastStartOffset " + lastOffset);
+                    }
+                    if (endOffset < 0) {
+                      throw new RuntimeException("term " + term + ": doc " + docID + ": pos " + pos + ": endOffset " + endOffset + " is out of bounds");
+                    }
+                    if (endOffset < startOffset) {
+                      throw new RuntimeException("term " + term + ": doc " + docID + ": pos " + pos + ": endOffset " + endOffset + " < startOffset " + startOffset);
+                    }
                   }
                   lastOffset = startOffset;
                 }
@@ -1193,12 +1201,12 @@ public class CheckIndex {
       }
 
       final Fields fields = reader.fields();
-      status = checkFields(fields, liveDocs, maxDoc, fieldInfos, true);
+      status = checkFields(fields, liveDocs, maxDoc, fieldInfos, true, false);
       if (liveDocs != null) {
         if (infoStream != null) {
           infoStream.print("    test (ignoring deletes): terms, freq, prox...");
         }
-        checkFields(fields, null, maxDoc, fieldInfos, true);
+        checkFields(fields, null, maxDoc, fieldInfos, true, false);
       }
     } catch (Throwable e) {
       msg("ERROR: " + e);
@@ -1415,10 +1423,10 @@ public class CheckIndex {
 
         if (tfv != null) {
           // First run with no deletions:
-          checkFields(tfv, null, 1, fieldInfos, false);
+          checkFields(tfv, null, 1, fieldInfos, false, true);
 
           // Again, with the one doc deleted:
-          checkFields(tfv, onlyDocIsDeleted, 1, fieldInfos, false);
+          checkFields(tfv, onlyDocIsDeleted, 1, fieldInfos, false, true);
 
           // Only agg stats if the doc is live:
           final boolean doStats = liveDocs == null || liveDocs.get(j);

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/MultiDocsAndPositionsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/MultiDocsAndPositionsEnum.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/MultiDocsAndPositionsEnum.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/MultiDocsAndPositionsEnum.java Fri Jul 20 01:01:39 2012
@@ -55,6 +55,7 @@ public final class MultiDocsAndPositions
       this.subs[i].slice = subs[i].slice;
     }
     upto = -1;
+    doc = -1;
     current = null;
     return this;
   }
@@ -69,6 +70,7 @@ public final class MultiDocsAndPositions
 
   @Override
   public int freq() throws IOException {
+    assert current != null;
     return current.freq();
   }
 

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/MultiDocsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/MultiDocsEnum.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/MultiDocsEnum.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/MultiDocsEnum.java Fri Jul 20 01:01:39 2012
@@ -52,6 +52,7 @@ public final class MultiDocsEnum extends
       this.subs[i].slice = subs[i].slice;
     }
     upto = -1;
+    doc = -1;
     current = null;
     return this;
   }

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java Fri Jul 20 01:01:39 2012
@@ -168,7 +168,7 @@ public class TieredMergePolicy extends M
 
   /** @see #setFloorSegmentMB */
   public double getFloorSegmentMB() {
-    return floorSegmentBytes/1024*1024.;
+    return floorSegmentBytes/(1024*1024.);
   }
 
   /** When forceMergeDeletes is called, we only merge away a

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java Fri Jul 20 01:01:39 2012
@@ -20,7 +20,6 @@ package org.apache.lucene.search;
 import org.apache.lucene.index.*;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.ConjunctionTermScorer.DocsAndFreqs;
-import org.apache.lucene.search.TermQuery.TermDocsEnumFactory;
 import org.apache.lucene.search.TermQuery.TermWeight;
 import org.apache.lucene.search.similarities.Similarity;
 import org.apache.lucene.search.similarities.Similarity.ExactSimScorer;
@@ -240,7 +239,7 @@ public class BooleanQuery extends Query 
       for (Iterator<Weight> wIter = weights.iterator(); wIter.hasNext();) {
         Weight w = wIter.next();
         BooleanClause c = cIter.next();
-        if (w.scorer(context, true, true, context.reader().getLiveDocs()) == null) {
+        if (w.scorer(context, true, true, false, false, false, context.reader().getLiveDocs()) == null) {
           if (c.isRequired()) {
             fail = true;
             Explanation r = new Explanation(0.0f, "no match on required clause (" + c.getQuery().toString() + ")");
@@ -303,11 +302,11 @@ public class BooleanQuery extends Query 
 
     @Override
     public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder,
-        boolean topScorer, Bits acceptDocs)
+        boolean topScorer, boolean needsPositions, boolean needsOffsets, boolean collectPositions, Bits acceptDocs)
         throws IOException {
       if (termConjunction) {
         // specialized scorer for term conjunctions
-        return createConjunctionTermScorer(context, acceptDocs);
+        return createConjunctionTermScorer(context, acceptDocs, needsPositions, needsOffsets, collectPositions);
       }
       List<Scorer> required = new ArrayList<Scorer>();
       List<Scorer> prohibited = new ArrayList<Scorer>();
@@ -315,7 +314,7 @@ public class BooleanQuery extends Query 
       Iterator<BooleanClause> cIter = clauses.iterator();
       for (Weight w  : weights) {
         BooleanClause c =  cIter.next();
-        Scorer subScorer = w.scorer(context, true, false, acceptDocs);
+        Scorer subScorer = w.scorer(context, true, false, needsPositions, needsOffsets, collectPositions, acceptDocs);
         if (subScorer == null) {
           if (c.isRequired()) {
             return null;
@@ -330,7 +329,7 @@ public class BooleanQuery extends Query 
       }
       
       // Check if we can return a BooleanScorer
-      if (!scoreDocsInOrder && topScorer && required.size() == 0) {
+      if (!scoreDocsInOrder && !needsPositions && topScorer && required.size() == 0) {
         return new BooleanScorer(this, disableCoord, minNrShouldMatch, optional, prohibited, maxCoord);
       }
       
@@ -345,10 +344,10 @@ public class BooleanQuery extends Query 
       }
       
       // Return a BooleanScorer2
-      return new BooleanScorer2(this, disableCoord, minNrShouldMatch, required, prohibited, optional, maxCoord);
+      return new BooleanScorer2(this, disableCoord, minNrShouldMatch, required, prohibited, optional, maxCoord, collectPositions);
     }
 
-    private Scorer createConjunctionTermScorer(AtomicReaderContext context, Bits acceptDocs)
+    private Scorer createConjunctionTermScorer(AtomicReaderContext context, Bits acceptDocs, boolean needsPositions, boolean needsOffsets, boolean collectPositions)
         throws IOException {
 
       // TODO: fix scorer API to specify "needsScores" up
@@ -358,47 +357,18 @@ public class BooleanQuery extends Query 
       final DocsAndFreqs[] docsAndFreqs = new DocsAndFreqs[weights.size()];
       for (int i = 0; i < docsAndFreqs.length; i++) {
         final TermWeight weight = (TermWeight) weights.get(i);
-        final TermsEnum termsEnum = weight.getTermsEnum(context);
-        if (termsEnum == null) {
+        final Scorer scorer = weight.scorer(context, true, false, needsPositions, needsOffsets, collectPositions, acceptDocs);
+        if (scorer == null) {
           return null;
         }
-        final ExactSimScorer docScorer = weight.createDocScorer(context);
-        final DocsEnum docsAndFreqsEnum = termsEnum.docs(acceptDocs, null, true);
-        if (docsAndFreqsEnum == null) {
-          // TODO: we could carry over TermState from the
-          // terms we already seek'd to, to save re-seeking
-          // to make the match-only scorer, but it's likely
-          // rare that BQ mixes terms from omitTf and
-          // non-omitTF fields:
-
-          // At least one sub cannot provide freqs; abort
-          // and fallback to full match-only scorer:
-          return createMatchOnlyConjunctionTermScorer(context, acceptDocs);
+        if (scorer instanceof TermScorer) {
+          docsAndFreqs[i] = new DocsAndFreqs((TermScorer) scorer);
+        } else {
+          docsAndFreqs[i] = new DocsAndFreqs((MatchOnlyTermScorer) scorer);
         }
-        TermDocsEnumFactory factory = new TermDocsEnumFactory(termsEnum, acceptDocs);
-        docsAndFreqs[i] = new DocsAndFreqs(termsEnum.docFreq(), docScorer, docsAndFreqsEnum, factory);
       }
       return new ConjunctionTermScorer(this, disableCoord ? 1.0f : coord(
-          docsAndFreqs.length, docsAndFreqs.length), docsAndFreqs);
-    }
-
-    private Scorer createMatchOnlyConjunctionTermScorer(AtomicReaderContext context, Bits acceptDocs)
-        throws IOException {
-
-      final DocsAndFreqs[] docsAndFreqs = new DocsAndFreqs[weights.size()];
-      for (int i = 0; i < docsAndFreqs.length; i++) {
-        final TermWeight weight = (TermWeight) weights.get(i);
-        final TermsEnum termsEnum = weight.getTermsEnum(context);
-        if (termsEnum == null) {
-          return null;
-        }
-        final ExactSimScorer docScorer = weight.createDocScorer(context);
-        TermDocsEnumFactory factory = new TermDocsEnumFactory(termsEnum, acceptDocs);
-        docsAndFreqs[i] = new DocsAndFreqs(termsEnum.docFreq(), docScorer, termsEnum.docs(acceptDocs, null, false), factory);
-      }
-
-      return new MatchOnlyConjunctionTermScorer(this, disableCoord ? 1.0f : coord(
-          docsAndFreqs.length, docsAndFreqs.length), docsAndFreqs);
+          docsAndFreqs.length, docsAndFreqs.length), collectPositions, docsAndFreqs);
     }
     
     @Override

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java Fri Jul 20 01:01:39 2012
@@ -141,7 +141,7 @@ final class BooleanScorer extends Scorer
     public float score() { return score; }
 
     @Override
-    public IntervalIterator positions(boolean needsPayloads, boolean needsOffsets, boolean collectPositions) throws IOException {
+    public IntervalIterator positions() throws IOException {
       return IntervalIterator.NO_MORE_POSITIONS;
     }
     
@@ -327,20 +327,13 @@ final class BooleanScorer extends Scorer
   }
 
   @Override
-  public IntervalIterator positions(boolean needsPayloads, boolean needsOffsets, boolean collectPositions) throws IOException {
-    final List<Scorer> scorers = new ArrayList<Scorer>();
-    SubScorer sub = this.scorers;
-    while(sub != null) {
-      if (!sub.prohibited) {
-        scorers.add(sub.scorer);
-      }
-      sub = sub.next;
-    }
-    if (this.minNrShouldMatch > 1) {
-      return new ConjunctionIntervalIterator(this,
-          collectPositions, this.minNrShouldMatch, BooleanIntervalIterator.pullIterators(needsPayloads, needsOffsets, collectPositions, scorers));
-    }
-    return new DisjunctionIntervalIterator(this, collectPositions,  BooleanIntervalIterator.pullIterators(needsPayloads, needsOffsets, collectPositions, scorers));
+  public IntervalIterator positions() throws IOException {
+    throw new UnsupportedOperationException("intervals are not available if docs are matched out of order");
+  }
+
+  @Override
+  public float freq() throws IOException {
+    return current.coord;
   }
 
   @Override
@@ -364,7 +357,8 @@ final class BooleanScorer extends Scorer
   public Collection<ChildScorer> getChildren() {
     List<ChildScorer> children = new ArrayList<ChildScorer>();
     for (SubScorer sub = scorers; sub != null; sub = sub.next) {
-      children.add(new ChildScorer(sub.scorer, sub.prohibited ? Occur.MUST_NOT.toString() : Occur.SHOULD.toString()));
+      // TODO: fix this if BQ ever sends us required clauses
+      children.add(new ChildScorer(sub.scorer, sub.prohibited ? "MUST_NOT" : "SHOULD"));
     }
     return children;
   }

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/BooleanScorer2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/BooleanScorer2.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/BooleanScorer2.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/BooleanScorer2.java Fri Jul 20 01:01:39 2012
@@ -88,7 +88,7 @@ class BooleanScorer2 extends Scorer {
    *          the list of optional scorers.
    */
   public BooleanScorer2(BooleanWeight weight, boolean disableCoord, int minNrShouldMatch,
-      List<Scorer> required, List<Scorer> prohibited, List<Scorer> optional, int maxCoord) throws IOException {
+      List<Scorer> required, List<Scorer> prohibited, List<Scorer> optional, int maxCoord, boolean collectPositions) throws IOException {
     super(weight);
     if (minNrShouldMatch < 0) {
       throw new IllegalArgumentException("Minimum number of optional scorers should not be negative");
@@ -102,7 +102,7 @@ class BooleanScorer2 extends Scorer {
     prohibitedScorers = prohibited;
     
     coordinator.init(disableCoord);
-    countingSumScorer = makeCountingSumScorer(disableCoord);
+    countingSumScorer = makeCountingSumScorer(disableCoord, collectPositions);
   }
   
   /** Count a scorer as a single match. */
@@ -132,6 +132,11 @@ class BooleanScorer2 extends Scorer {
     }
 
     @Override
+    public float freq() throws IOException {
+      return 1;
+    }
+
+    @Override
     public int docID() {
       return scorer.docID();
     }
@@ -147,15 +152,15 @@ class BooleanScorer2 extends Scorer {
     }
     
     @Override
-    public IntervalIterator positions(boolean needsPayloads, boolean needsOffsets, boolean collectPositions) throws IOException {
-      return scorer.positions(needsPayloads, needsOffsets, false);
+    public IntervalIterator positions() throws IOException {
+      return scorer.positions();
     }
   }
 
   private Scorer countingDisjunctionSumScorer(final List<Scorer> scorers,
-      int minNrShouldMatch) throws IOException {
+      int minNrShouldMatch, boolean collectPositions) throws IOException {
     // each scorer from the list counted as a single matcher
-    return new DisjunctionSumScorer(weight, scorers, minNrShouldMatch) {
+    return new DisjunctionSumScorer(weight, scorers, minNrShouldMatch, collectPositions) {
       private int lastScoredDoc = -1;
       // Save the score of lastScoredDoc, so that we don't compute it more than
       // once in score().
@@ -174,11 +179,11 @@ class BooleanScorer2 extends Scorer {
     };
   }
 
-  private Scorer countingConjunctionSumScorer(boolean disableCoord,
+  private Scorer countingConjunctionSumScorer(boolean disableCoord, boolean collectPositions,
                                               List<Scorer> requiredScorers) throws IOException {
     // each scorer from the list counted as a single matcher
     final int requiredNrMatchers = requiredScorers.size();
-    return new ConjunctionScorer(weight, disableCoord ? 1.0f : ((BooleanWeight)weight).coord(requiredScorers.size(), requiredScorers.size()), requiredScorers) {
+    return new ConjunctionScorer(weight, disableCoord ? 1.0f : ((BooleanWeight)weight).coord(requiredScorers.size(), requiredScorers.size()), collectPositions, requiredScorers) {
       private int lastScoredDoc = -1;
       // Save the score of lastScoredDoc, so that we don't compute it more than
       // once in score().
@@ -201,9 +206,9 @@ class BooleanScorer2 extends Scorer {
     };
   }
 
-  private Scorer dualConjunctionSumScorer(boolean disableCoord,
+  private Scorer dualConjunctionSumScorer(boolean disableCoord, boolean collectPositions,
                                           Scorer req1, Scorer req2) throws IOException { // non counting.
-    return new ConjunctionScorer(weight, disableCoord ? 1.0f : ((BooleanWeight)weight).coord(2, 2), req1, req2);
+    return new ConjunctionScorer(weight, disableCoord ? 1.0f : ((BooleanWeight)weight).coord(2, 2), collectPositions, req1, req2);
     // All scorers match, so defaultSimilarity always has 1 as
     // the coordination factor.
     // Therefore the sum of the scores of two scorers
@@ -213,51 +218,52 @@ class BooleanScorer2 extends Scorer {
   /** Returns the scorer to be used for match counting and score summing.
    * Uses requiredScorers, optionalScorers and prohibitedScorers.
    */
-  private Scorer makeCountingSumScorer(boolean disableCoord) throws IOException { // each scorer counted as a single matcher
+  private Scorer makeCountingSumScorer(boolean disableCoord, boolean collectPositions) throws IOException { // each scorer counted as a single matcher
     return (requiredScorers.size() == 0)
-      ? makeCountingSumScorerNoReq(disableCoord)
-      : makeCountingSumScorerSomeReq(disableCoord);
+      ? makeCountingSumScorerNoReq(disableCoord, collectPositions)
+      : makeCountingSumScorerSomeReq(disableCoord, collectPositions);
   }
 
-  private Scorer makeCountingSumScorerNoReq(boolean disableCoord) throws IOException { // No required scorers
+  private Scorer makeCountingSumScorerNoReq(boolean disableCoord, boolean collectPositions) throws IOException { // No required scorers
     // minNrShouldMatch optional scorers are required, but at least 1
     int nrOptRequired = (minNrShouldMatch < 1) ? 1 : minNrShouldMatch;
     Scorer requiredCountingSumScorer;
     if (optionalScorers.size() > nrOptRequired)
-      requiredCountingSumScorer = countingDisjunctionSumScorer(optionalScorers, nrOptRequired);
+      requiredCountingSumScorer = countingDisjunctionSumScorer(optionalScorers, nrOptRequired, collectPositions);
     else if (optionalScorers.size() == 1)
       requiredCountingSumScorer = new SingleMatchScorer(optionalScorers.get(0));
     else {
-      requiredCountingSumScorer = countingConjunctionSumScorer(disableCoord, optionalScorers);
+      requiredCountingSumScorer = countingConjunctionSumScorer(disableCoord, collectPositions, optionalScorers);
     }
-    return addProhibitedScorers(requiredCountingSumScorer);
+    return addProhibitedScorers(requiredCountingSumScorer, collectPositions);
   }
 
-  private Scorer makeCountingSumScorerSomeReq(boolean disableCoord) throws IOException { // At least one required scorer.
+  private Scorer makeCountingSumScorerSomeReq(boolean disableCoord, boolean collectPositions) throws IOException { // At least one required scorer.
     if (optionalScorers.size() == minNrShouldMatch) { // all optional scorers also required.
       ArrayList<Scorer> allReq = new ArrayList<Scorer>(requiredScorers);
       allReq.addAll(optionalScorers);
-      return addProhibitedScorers(countingConjunctionSumScorer(disableCoord, allReq));
+      return addProhibitedScorers(countingConjunctionSumScorer(disableCoord, collectPositions, allReq), collectPositions);
     } else { // optionalScorers.size() > minNrShouldMatch, and at least one required scorer
       Scorer requiredCountingSumScorer =
             requiredScorers.size() == 1
             ? new SingleMatchScorer(requiredScorers.get(0))
-            : countingConjunctionSumScorer(disableCoord, requiredScorers);
+            : countingConjunctionSumScorer(disableCoord, collectPositions, requiredScorers);
       if (minNrShouldMatch > 0) { // use a required disjunction scorer over the optional scorers
         return addProhibitedScorers( 
                       dualConjunctionSumScorer( // non counting
-                              disableCoord,
+                              disableCoord, 
+                              collectPositions,
                               requiredCountingSumScorer,
                               countingDisjunctionSumScorer(
                                       optionalScorers,
-                                      minNrShouldMatch)));
+                                      minNrShouldMatch, collectPositions)), collectPositions);
       } else { // minNrShouldMatch == 0
         return new ReqOptSumScorer(
-                      addProhibitedScorers(requiredCountingSumScorer),
+                      addProhibitedScorers(requiredCountingSumScorer, collectPositions),
                       optionalScorers.size() == 1
                         ? new SingleMatchScorer(optionalScorers.get(0))
                         // require 1 in combined, optional scorer.
-                        : countingDisjunctionSumScorer(optionalScorers, 1));
+                        : countingDisjunctionSumScorer(optionalScorers, 1, collectPositions), collectPositions);
       }
     }
   }
@@ -266,14 +272,14 @@ class BooleanScorer2 extends Scorer {
    * Uses the given required scorer and the prohibitedScorers.
    * @param requiredCountingSumScorer A required scorer already built.
    */
-  private Scorer addProhibitedScorers(Scorer requiredCountingSumScorer) throws IOException
+  private Scorer addProhibitedScorers(Scorer requiredCountingSumScorer, boolean collectPositions) throws IOException
   {
     return (prohibitedScorers.size() == 0)
           ? requiredCountingSumScorer // no prohibited
           : new ReqExclScorer(requiredCountingSumScorer,
                               ((prohibitedScorers.size() == 1)
                                 ? prohibitedScorers.get(0)
-                                : new DisjunctionSumScorer(weight, prohibitedScorers)));
+                                : new DisjunctionSumScorer(weight, collectPositions, prohibitedScorers)), collectPositions);
   }
 
   /** Scores and collects all matching documents.
@@ -316,8 +322,8 @@ class BooleanScorer2 extends Scorer {
   }
 
   @Override
-  public float freq() {
-    return coordinator.nrMatchers;
+  public float freq() throws IOException {
+    return countingSumScorer.freq();
   }
 
   @Override
@@ -326,21 +332,21 @@ class BooleanScorer2 extends Scorer {
   }
   
   @Override
-  public IntervalIterator positions(boolean needsPayloads, boolean needsOffsets, boolean collectPositions) throws IOException {
-    return countingSumScorer.positions(needsPayloads, needsOffsets, collectPositions);
+  public IntervalIterator positions() throws IOException {
+    return countingSumScorer.positions();
   }
 
   @Override
   public Collection<ChildScorer> getChildren() {
     ArrayList<ChildScorer> children = new ArrayList<ChildScorer>();
     for (Scorer s : optionalScorers) {
-      children.add(new ChildScorer(s, Occur.SHOULD.toString()));
+      children.add(new ChildScorer(s, "SHOULD"));
     }
     for (Scorer s : prohibitedScorers) {
-      children.add(new ChildScorer(s, Occur.MUST_NOT.toString()));
+      children.add(new ChildScorer(s, "MUST_NOT"));
     }
     for (Scorer s : requiredScorers) {
-      children.add(new ChildScorer(s, Occur.MUST.toString()));
+      children.add(new ChildScorer(s, "MUST"));
     }
     return children;
   }

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/CachingCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/CachingCollector.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/CachingCollector.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/CachingCollector.java Fri Jul 20 01:01:39 2012
@@ -92,7 +92,7 @@ public abstract class CachingCollector e
     public final int nextDoc() { throw new UnsupportedOperationException(); }
 
     @Override
-    public IntervalIterator positions(boolean needsPayloads, boolean needsOffsets, boolean collectPositions) throws IOException { throw new UnsupportedOperationException(); }
+    public IntervalIterator positions() throws IOException { throw new UnsupportedOperationException(); }
     }
 
   // A CachingCollector which caches scores

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/Collector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/Collector.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/Collector.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/Collector.java Fri Jul 20 01:01:39 2012
@@ -175,6 +175,9 @@ public abstract class Collector {
   
   public boolean needsPositions() { return false; }
   
-  public boolean needsPayloads() { return false; }
+  public boolean needsOffsets() { return false; }
+
+  public boolean collectsPositions() { return false; }
+
   
 }

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java Fri Jul 20 01:01:39 2012
@@ -18,6 +18,7 @@ package org.apache.lucene.search;
  */
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Comparator;
 
@@ -33,13 +34,15 @@ class ConjunctionScorer extends Scorer {
   private final Scorer[] scorers;
   private final float coord;
   private int lastDoc = -1;
+  private final boolean collectPositions;
 
-  public ConjunctionScorer(Weight weight, float coord, Collection<Scorer> scorers) throws IOException {
-    this(weight, coord, scorers.toArray(new Scorer[scorers.size()]));
+  public ConjunctionScorer(Weight weight, float coord, boolean collectPositions, Collection<Scorer> scorers) throws IOException {
+    this(weight, coord, collectPositions, scorers.toArray(new Scorer[scorers.size()]));
   }
   
-  public ConjunctionScorer(Weight weight, float coord, Scorer... scorers) throws IOException {
+  public ConjunctionScorer(Weight weight, float coord, boolean collectPositions, Scorer... scorers) throws IOException {
     super(weight);
+    this.collectPositions = collectPositions;
     scorersOrdered = new Scorer[scorers.length];
     System.arraycopy(scorers, 0, scorersOrdered, 0, scorers.length);
     this.scorers = scorers;
@@ -145,12 +148,26 @@ class ConjunctionScorer extends Scorer {
   }
   
   @Override
-  public IntervalIterator positions(boolean needsPayloads, boolean needsOffsets, boolean collectPositions) throws IOException {
+  public IntervalIterator positions() throws IOException {
     if (scorersOrdered == null) {
       throw new IllegalStateException("no positions requested for this scorer");
     }
       // only created if needed for this scorer - no penalty for non-positional queries
-    return new ConjunctionIntervalIterator(this, collectPositions, BooleanIntervalIterator.pullIterators(needsPayloads, needsOffsets, collectPositions, scorersOrdered));
+    return new ConjunctionIntervalIterator(this, collectPositions, BooleanIntervalIterator.pullIterators(scorersOrdered));
   }
 
+
+  @Override
+  public float freq() throws IOException {
+    return scorers.length;
+  }
+
+  @Override
+  public Collection<ChildScorer> getChildren() {
+    ArrayList<ChildScorer> children = new ArrayList<ChildScorer>(scorers.length);
+    for (Scorer scorer : scorers) {
+      children.add(new ChildScorer(scorer, "MUST"));
+    }
+    return children;
+  }
 }

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/ConjunctionTermScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/ConjunctionTermScorer.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/ConjunctionTermScorer.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/ConjunctionTermScorer.java Fri Jul 20 01:01:39 2012
@@ -18,14 +18,15 @@ package org.apache.lucene.search;
  */
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Comparator;
 
+import org.apache.lucene.index.DocsAndPositionsEnum;
 import org.apache.lucene.index.DocsEnum;
-import org.apache.lucene.search.TermQuery.TermDocsEnumFactory;
 import org.apache.lucene.search.positions.ConjunctionIntervalIterator;
 import org.apache.lucene.search.positions.IntervalIterator;
 import org.apache.lucene.search.positions.TermIntervalIterator;
-import org.apache.lucene.search.similarities.Similarity.ExactSimScorer;
 import org.apache.lucene.util.ArrayUtil;
 
 
@@ -36,10 +37,12 @@ class ConjunctionTermScorer extends Scor
   protected final DocsAndFreqs[] docsAndFreqs;
   private final DocsAndFreqs lead;
   private DocsAndFreqs[] origDocsAndFreqs;
+  private final boolean collectPositions;
 
-  ConjunctionTermScorer(Weight weight, float coord,
+  ConjunctionTermScorer(Weight weight, float coord, boolean collectPositions,
       DocsAndFreqs[] docsAndFreqs) {
     super(weight);
+    this.collectPositions = collectPositions;
     this.coord = coord;
     this.docsAndFreqs = docsAndFreqs;
     this.origDocsAndFreqs = new DocsAndFreqs[docsAndFreqs.length];
@@ -99,34 +102,54 @@ class ConjunctionTermScorer extends Scor
   public float score() throws IOException {
     float sum = 0.0f;
     for (DocsAndFreqs docs : docsAndFreqs) {
-      sum += docs.docScorer.score(lastDoc, docs.docs.freq());
+      sum += docs.scorer.score();
     }
     return sum * coord;
   }
+  
+  @Override
+  public float freq() {
+    return docsAndFreqs.length;
+  }
+
+  @Override
+  public Collection<ChildScorer> getChildren() {
+    ArrayList<ChildScorer> children = new ArrayList<ChildScorer>(docsAndFreqs.length);
+    for (DocsAndFreqs docs : docsAndFreqs) {
+      children.add(new ChildScorer(docs.scorer, "MUST"));
+    }
+    return children;
+  }
 
   static final class DocsAndFreqs {
-    //final DocsEnum docsAndFreqs;
     final DocsEnum docs;
     final int docFreq;
-    final ExactSimScorer docScorer;
+    final Scorer scorer;
     int doc = -1;
-    private final TermDocsEnumFactory factory;
 
-    DocsAndFreqs( int docFreq, ExactSimScorer docScorer, DocsEnum docs, TermDocsEnumFactory factory) throws IOException {
-      //this.docsAndFreqs = factory.docsAndFreqsEnum();
+    DocsAndFreqs(TermScorer termScorer) {
+      this(termScorer, termScorer.getDocsEnum(), termScorer.getDocFreq());
+    }
+    
+    DocsAndFreqs(MatchOnlyTermScorer termScorer) {
+      this(termScorer, termScorer.getDocsEnum(), termScorer.getDocFreq());
+    }
+    
+    DocsAndFreqs(Scorer scorer, DocsEnum docs, int docFreq) {
       this.docs = docs;
       this.docFreq = docFreq;
-      this.docScorer = docScorer;
-      this.factory = factory;
+      this.scorer = scorer;
     }
   }
 
   @Override
-  public IntervalIterator positions(boolean needsPayloads, boolean needsOffsets, boolean collectPositions) throws IOException {
+  public IntervalIterator positions() throws IOException {
+    
     TermIntervalIterator[] positionIters = new TermIntervalIterator[origDocsAndFreqs.length];
     for (int i = 0; i < positionIters.length; i++) {
       DocsAndFreqs d = origDocsAndFreqs[i];
-      positionIters[i] = new TermIntervalIterator(this, d.factory.docsAndPositionsEnum(needsOffsets), needsPayloads, collectPositions);
+      assert d.docs instanceof DocsAndPositionsEnum;
+      positionIters[i] = new TermIntervalIterator(this, (DocsAndPositionsEnum)d.docs, false, collectPositions);
     }
     return new ConjunctionIntervalIterator(this, collectPositions, positionIters);
   }

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java Fri Jul 20 01:01:39 2012
@@ -123,7 +123,7 @@ public class ConstantScoreQuery extends 
 
     @Override
     public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder,
-        boolean topScorer, final Bits acceptDocs) throws IOException {
+        boolean topScorer, boolean needsPositions, boolean needsOffsets, boolean collectPositions, final Bits acceptDocs) throws IOException {
       final DocIdSetIterator disi;
       if (filter != null) {
         assert query == null;
@@ -134,7 +134,7 @@ public class ConstantScoreQuery extends 
         disi = dis.iterator();
       } else {
         assert query != null && innerWeight != null;
-        disi = innerWeight.scorer(context, scoreDocsInOrder, topScorer, acceptDocs);
+        disi = innerWeight.scorer(context, scoreDocsInOrder, topScorer, needsPositions, needsOffsets, collectPositions, acceptDocs);
       }
 
       if (disi == null) {
@@ -150,7 +150,7 @@ public class ConstantScoreQuery extends 
 
     @Override
     public Explanation explain(AtomicReaderContext context, int doc) throws IOException {
-      final Scorer cs = scorer(context, true, false, context.reader().getLiveDocs());
+      final Scorer cs = scorer(context, true, false, false, false, false, context.reader().getLiveDocs());
       final boolean exists = (cs != null && cs.advance(doc) == doc);
 
       final ComplexExplanation result = new ComplexExplanation();
@@ -195,6 +195,11 @@ public class ConstantScoreQuery extends 
     }
 
     @Override
+    public float freq() throws IOException {
+      return 1;
+    }
+
+    @Override
     public int advance(int target) throws IOException {
       return docIdSetIterator.advance(target);
     }
@@ -229,8 +234,8 @@ public class ConstantScoreQuery extends 
         }
         
         @Override
-        public boolean needsPayloads() {
-          return collector.needsPayloads();
+        public boolean needsOffsets() {
+          return collector.needsOffsets();
         }
       };
     }
@@ -256,9 +261,9 @@ public class ConstantScoreQuery extends 
     }
         
     @Override
-    public IntervalIterator positions(boolean needsPayloads, boolean needsOffsets, boolean collectPositions) throws IOException {
+    public IntervalIterator positions() throws IOException {
       if (docIdSetIterator instanceof Scorer) {
-        return ((Scorer) docIdSetIterator).positions(needsPayloads, needsOffsets, false);
+        return ((Scorer) docIdSetIterator).positions();
       } else {
         throw new UnsupportedOperationException("positions are only supported on Scorer subclasses");
       }