lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rm...@apache.org
Subject svn commit: r1446801 [1/2] - in /lucene/dev/branches/lucene4765: ./ dev-tools/ dev-tools/maven/ lucene/ lucene/core/ lucene/core/src/java/org/apache/lucene/index/ lucene/core/src/java/org/apache/lucene/search/ lucene/core/src/java/org/apache/lucene/sto...
Date Fri, 15 Feb 2013 22:58:28 GMT
Author: rmuir
Date: Fri Feb 15 22:58:27 2013
New Revision: 1446801

URL: http://svn.apache.org/r1446801
Log:
Merged /lucene/dev/trunk:r1445907-1446753

Added:
    lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/search/TestSort2.java
      - copied unchanged from r1446753, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestSort2.java
    lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/search/TestSortDocValues.java
      - copied unchanged from r1446753, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestSortDocValues.java
    lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java
      - copied unchanged from r1446753, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java
    lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java
      - copied unchanged from r1446753, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java
    lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/search/TestTotalHitCountCollector.java
      - copied unchanged from r1446753, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestTotalHitCountCollector.java
    lucene/dev/branches/lucene4765/solr/example/etc/create-solrtest.keystore.sh
      - copied unchanged from r1446753, lucene/dev/trunk/solr/example/etc/create-solrtest.keystore.sh
    lucene/dev/branches/lucene4765/solr/example/etc/solrtest.keystore
      - copied unchanged from r1446753, lucene/dev/trunk/solr/example/etc/solrtest.keystore
    lucene/dev/branches/lucene4765/solr/licenses/httpclient-4.2.3.jar.sha1
      - copied unchanged from r1446753, lucene/dev/trunk/solr/licenses/httpclient-4.2.3.jar.sha1
    lucene/dev/branches/lucene4765/solr/licenses/httpcore-4.2.2.jar.sha1
      - copied unchanged from r1446753, lucene/dev/trunk/solr/licenses/httpcore-4.2.2.jar.sha1
    lucene/dev/branches/lucene4765/solr/licenses/httpmime-4.2.3.jar.sha1
      - copied unchanged from r1446753, lucene/dev/trunk/solr/licenses/httpmime-4.2.3.jar.sha1
Removed:
    lucene/dev/branches/lucene4765/solr/licenses/httpclient-4.1.3.jar.sha1
    lucene/dev/branches/lucene4765/solr/licenses/httpcore-4.1.4.jar.sha1
    lucene/dev/branches/lucene4765/solr/licenses/httpmime-4.1.3.jar.sha1
Modified:
    lucene/dev/branches/lucene4765/   (props changed)
    lucene/dev/branches/lucene4765/dev-tools/   (props changed)
    lucene/dev/branches/lucene4765/dev-tools/maven/pom.xml.template
    lucene/dev/branches/lucene4765/lucene/   (props changed)
    lucene/dev/branches/lucene4765/lucene/CHANGES.txt   (contents, props changed)
    lucene/dev/branches/lucene4765/lucene/core/   (props changed)
    lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
    lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
    lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java
    lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java
    lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java
    lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/search/FieldCache.java
    lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/store/RateLimitedDirectoryWrapper.java
    lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
    lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/search/TestSort.java
    lucene/dev/branches/lucene4765/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java
    lucene/dev/branches/lucene4765/lucene/spatial/   (props changed)
    lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java
    lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java
    lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/TermQueryPrefixTreeStrategy.java
    lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java
    lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java
    lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java
    lucene/dev/branches/lucene4765/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/JtsPolygonTest.java
    lucene/dev/branches/lucene4765/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/SpatialOpRecursivePrefixTreeTest.java
    lucene/dev/branches/lucene4765/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTreeTest.java
    lucene/dev/branches/lucene4765/lucene/test-framework/   (props changed)
    lucene/dev/branches/lucene4765/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java
    lucene/dev/branches/lucene4765/lucene/tools/   (props changed)
    lucene/dev/branches/lucene4765/lucene/tools/junit4/tests.policy
    lucene/dev/branches/lucene4765/solr/   (props changed)
    lucene/dev/branches/lucene4765/solr/CHANGES.txt   (contents, props changed)
    lucene/dev/branches/lucene4765/solr/core/   (props changed)
    lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
    lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
    lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/core/SolrCore.java
    lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/update/SolrCoreState.java
    lucene/dev/branches/lucene4765/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java
    lucene/dev/branches/lucene4765/solr/example/   (props changed)
    lucene/dev/branches/lucene4765/solr/example/etc/jetty.xml
    lucene/dev/branches/lucene4765/solr/licenses/   (props changed)
    lucene/dev/branches/lucene4765/solr/solrj/   (props changed)
    lucene/dev/branches/lucene4765/solr/solrj/ivy.xml
    lucene/dev/branches/lucene4765/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrServer.java
    lucene/dev/branches/lucene4765/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java
    lucene/dev/branches/lucene4765/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleJettyTest.java
    lucene/dev/branches/lucene4765/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpClientUtilTest.java
    lucene/dev/branches/lucene4765/solr/solrj/src/test/org/apache/solr/client/solrj/impl/LBHttpSolrServerTest.java
    lucene/dev/branches/lucene4765/solr/test-framework/   (props changed)
    lucene/dev/branches/lucene4765/solr/test-framework/src/java/org/apache/solr/SolrJettyTestBase.java

Modified: lucene/dev/branches/lucene4765/dev-tools/maven/pom.xml.template
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/dev-tools/maven/pom.xml.template?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/dev-tools/maven/pom.xml.template (original)
+++ lucene/dev/branches/lucene4765/dev-tools/maven/pom.xml.template Fri Feb 15 22:58:27 2013
@@ -48,7 +48,7 @@
     <jetty.version>8.1.8.v20121106</jetty.version>
     <slf4j.version>1.6.4</slf4j.version>
     <tika.version>1.2</tika.version>
-    <httpcomponents.version>4.1.3</httpcomponents.version>
+    <httpcomponents.version>4.2.3</httpcomponents.version>
 
     <!-- RandomizedTesting library system properties -->
     <tests.iters>1</tests.iters>

Modified: lucene/dev/branches/lucene4765/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/CHANGES.txt?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/lucene4765/lucene/CHANGES.txt Fri Feb 15 22:58:27 2013
@@ -168,6 +168,9 @@ New Features
   is faster than others, however consumes much more RAM.
   (Michael McCandless, Shai Erera)
   
+* LUCENE-4778: Add a getter for the delegate in RateLimitedDirectoryWrapper.
+  (Mark Miller)
+ 
 API Changes
 
 * LUCENE-4709: FacetResultNode no longer has a residue field. (Shai Erera)
@@ -216,6 +219,19 @@ Bug Fixes
   cases, for example if you had an index with more than 260M documents and a 
   VAR_INT field.  (Simon Willnauer, Adrien Grand, Mike McCandless, Robert Muir)
 
+* LUCENE-4775: Remove SegmentInfo.sizeInBytes() and make
+  MergePolicy.OneMerge.totalBytesSize thread safe (Josh Bronson via
+  Robert Muir, Mike McCandless)
+
+* LUCENE-4770: If spatial's TermQueryPrefixTreeStrategy was used to search
+  indexed non-point shapes, then there was an edge case where a query should
+  find a shape but it didn't. The fix is the removal of an optimization that
+  simplifies some leaf cells into a parent. The index data for such a field is
+  now ~20% larger. This optimization is still done for the query shape, and for
+  indexed data for RecursivePrefixTreeStrategy. Furthermore, this optimization
+  is enhanced to roll up beyond the bottom cell level. (David Smiley,
+  Florian Schilling)
+
 Documentation
 
 * LUCENE-4718: Fixed documentation of oal.queryparser.classic.

Modified: lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (original)
+++ lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java Fri Feb 15 22:58:27 2013
@@ -520,7 +520,7 @@ class DocumentsWriterPerThread {
       }
 
       if (infoStream.isEnabled("DWPT")) {
-        final double newSegmentSize = segmentInfo.sizeInBytes()/1024./1024.;
+        final double newSegmentSize = segmentInfoPerCommit.sizeInBytes()/1024./1024.;
         infoStream.message("DWPT", "flushed: segment=" + segmentInfo.name + 
                 " ramUsed=" + nf.format(startMBUsed) + " MB" +
                 " newFlushedSize(includes docstores)=" + nf.format(newSegmentSize) + " MB" +
@@ -557,7 +557,7 @@ class DocumentsWriterPerThread {
 
     IndexWriter.setDiagnostics(newSegment.info, "flush");
     
-    IOContext context = new IOContext(new FlushInfo(newSegment.info.getDocCount(), newSegment.info.sizeInBytes()));
+    IOContext context = new IOContext(new FlushInfo(newSegment.info.getDocCount(), newSegment.sizeInBytes()));
 
     boolean success = false;
     try {

Modified: lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Fri Feb 15 22:58:27 2013
@@ -2301,7 +2301,7 @@ public class IndexWriter implements Clos
               infoStream.message("IW", "addIndexes: process segment origName=" + info.info.name + " newName=" + newSegName + " info=" + info);
             }
 
-            IOContext context = new IOContext(new MergeInfo(info.info.getDocCount(), info.info.sizeInBytes(), true, -1));
+            IOContext context = new IOContext(new MergeInfo(info.info.getDocCount(), info.sizeInBytes(), true, -1));
 
             for(FieldInfo fi : getFieldInfos(info.info)) {
               globalFieldNumberMap.addOrGet(fi.name, fi.number, fi.getDocValuesType());
@@ -3458,7 +3458,8 @@ public class IndexWriter implements Clos
         final int delCount = numDeletedDocs(info);
         assert delCount <= info.info.getDocCount();
         final double delRatio = ((double) delCount)/info.info.getDocCount();
-        merge.estimatedMergeBytes += info.info.sizeInBytes() * (1.0 - delRatio);
+        merge.estimatedMergeBytes += info.sizeInBytes() * (1.0 - delRatio);
+        merge.totalMergeBytes += info.sizeInBytes();
       }
     }
   }
@@ -3759,7 +3760,7 @@ public class IndexWriter implements Clos
       // lost... 
 
       if (infoStream.isEnabled("IW")) {
-        infoStream.message("IW", String.format(Locale.ROOT, "merged segment size=%.3f MB vs estimate=%.3f MB", merge.info.info.sizeInBytes()/1024./1024., merge.estimatedMergeBytes/1024/1024.));
+        infoStream.message("IW", String.format(Locale.ROOT, "merged segment size=%.3f MB vs estimate=%.3f MB", merge.info.sizeInBytes()/1024./1024., merge.estimatedMergeBytes/1024/1024.));
       }
 
       final IndexReaderWarmer mergedSegmentWarmer = config.getMergedSegmentWarmer();

Modified: lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java (original)
+++ lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java Fri Feb 15 22:58:27 2013
@@ -19,7 +19,6 @@ package org.apache.lucene.index;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
@@ -74,7 +73,11 @@ public abstract class MergePolicy implem
     int maxNumSegments = -1;        // used by IndexWriter
 
     /** Estimated size in bytes of the merged segment. */
-    public long estimatedMergeBytes;       // used by IndexWriter
+    public volatile long estimatedMergeBytes;       // used by IndexWriter
+
+    // Sum of sizeInBytes of all SegmentInfos; set by IW.mergeInit
+    volatile long totalMergeBytes;
+
     List<SegmentReader> readers;        // used by IndexWriter
 
     /** Segments to be merged. */
@@ -187,14 +190,12 @@ public abstract class MergePolicy implem
     
     /**
      * Returns the total size in bytes of this merge. Note that this does not
-     * indicate the size of the merged segment, but the input total size.
-     * */
+     * indicate the size of the merged segment, but the
+     * input total size. This is only set once the merge is
+     * initialized by IndexWriter.
+     */
     public long totalBytesSize() throws IOException {
-      long total = 0;
-      for (SegmentInfoPerCommit info : segments) {
-        total += info.info.sizeInBytes();
-      }
-      return total;
+      return totalMergeBytes;
     }
 
     /**

Modified: lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java (original)
+++ lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java Fri Feb 15 22:58:27 2013
@@ -18,7 +18,6 @@ package org.apache.lucene.index;
  */
 
 
-import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
@@ -57,8 +56,6 @@ public final class SegmentInfo {
 
   private boolean isCompoundFile;
 
-  private volatile long sizeInBytes = -1;         // total byte size of all files (computed on demand)
-
   private Codec codec;
 
   private Map<String,String> diagnostics;
@@ -101,23 +98,6 @@ public final class SegmentInfo {
   }
 
   /**
-   * Returns total size in bytes of all of files used by
-   * this segment.  Note that this will not include any live
-   * docs for the segment; to include that use {@link
-   * SegmentInfoPerCommit#sizeInBytes()} instead.
-   */
-  public long sizeInBytes() throws IOException {
-    if (sizeInBytes == -1) {
-      long sum = 0;
-      for (final String fileName : files()) {
-        sum += dir.fileLength(fileName);
-      }
-      sizeInBytes = sum;
-    }
-    return sizeInBytes;
-  }
-
-  /**
    * Mark whether this segment is stored as a compound file.
    *
    * @param isCompoundFile true if this is a compound file;
@@ -254,7 +234,6 @@ public final class SegmentInfo {
   public void setFiles(Set<String> files) {
     checkFileNames(files);
     setFiles = files;
-    sizeInBytes = -1;
   }
 
   /** Add these files to the set of files written for this
@@ -262,7 +241,6 @@ public final class SegmentInfo {
   public void addFiles(Collection<String> files) {
     checkFileNames(files);
     setFiles.addAll(files);
-    sizeInBytes = -1;
   }
 
   /** Add this file to the set of files written for this
@@ -270,7 +248,6 @@ public final class SegmentInfo {
   public void addFile(String file) {
     checkFileNames(Collections.singleton(file));
     setFiles.add(file);
-    sizeInBytes = -1;
   }
   
   private void checkFileNames(Collection<String> files) {

Modified: lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java (original)
+++ lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java Fri Feb 15 22:58:27 2013
@@ -374,7 +374,7 @@ public class TieredMergePolicy extends M
       for(int idx = tooBigCount; idx<infosSorted.size(); idx++) {
         final SegmentInfoPerCommit info = infosSorted.get(idx);
         if (merging.contains(info)) {
-          mergingBytes += info.info.sizeInBytes();
+          mergingBytes += info.sizeInBytes();
         } else if (!toBeMerged.contains(info)) {
           eligible.add(info);
         }
@@ -470,7 +470,7 @@ public class TieredMergePolicy extends M
       final long segBytes = size(info);
       totAfterMergeBytes += segBytes;
       totAfterMergeBytesFloored += floorSize(segBytes);
-      totBeforeMergeBytes += info.info.sizeInBytes();
+      totBeforeMergeBytes += info.sizeInBytes();
     }
 
     // Measure "skew" of the merge, which can range
@@ -670,7 +670,7 @@ public class TieredMergePolicy extends M
 
   // Segment size in bytes, pro-rated by % deleted
   private long size(SegmentInfoPerCommit info) throws IOException {
-    final long byteSize = info.info.sizeInBytes();    
+    final long byteSize = info.sizeInBytes();    
     final int delCount = writer.get().numDeletedDocs(info);
     final double delRatio = (info.info.getDocCount() <= 0 ? 0.0f : ((double)delCount / (double)info.info.getDocCount()));    
     assert delRatio <= 1.0;

Modified: lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/search/FieldCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/search/FieldCache.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/search/FieldCache.java (original)
+++ lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/search/FieldCache.java Fri Feb 15 22:58:27 2013
@@ -91,7 +91,7 @@ public interface FieldCache {
     };
   }
 
-  /** Field values as 32-bit signed long integers */
+  /** Field values as 64-bit signed long integers */
   public static abstract class Longs {
     /** Return an long representation of this field's value. */
     public abstract long get(int docID);

Modified: lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/store/RateLimitedDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/store/RateLimitedDirectoryWrapper.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/store/RateLimitedDirectoryWrapper.java (original)
+++ lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/store/RateLimitedDirectoryWrapper.java Fri Feb 15 22:58:27 2013
@@ -40,6 +40,10 @@ public final class RateLimitedDirectoryW
     this.delegate = wrapped;
   }
   
+  public Directory getDelegate() {
+    return delegate;
+  }
+  
   @Override
   public String[] listAll() throws IOException {
     ensureOpen();

Modified: lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java (original)
+++ lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java Fri Feb 15 22:58:27 2013
@@ -324,4 +324,40 @@ public class TestConcurrentMergeSchedule
     w.close(false);
     dir.close();
   }
+
+
+  private static class TrackingCMS extends ConcurrentMergeScheduler {
+    long totMergedBytes;
+
+    public TrackingCMS() {
+      setMaxMergeCount(5);
+      setMaxThreadCount(5);
+    }
+
+    @Override
+    public void doMerge(MergePolicy.OneMerge merge) throws IOException {
+      totMergedBytes += merge.totalBytesSize();
+      super.doMerge(merge);
+    }
+  }
+
+  public void testTotalBytesSize() throws Exception {
+    Directory d = newDirectory();
+    IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+    iwc.setMaxBufferedDocs(5);
+    iwc.setMergeScheduler(new TrackingCMS());
+    RandomIndexWriter w = new RandomIndexWriter(random(), d);
+    for(int i=0;i<100000;i++) {
+      Document doc = new Document();
+      doc.add(newStringField("id", ""+i, Field.Store.NO));
+      doc.add(newTextField("field", "here is some text", Field.Store.NO));
+      w.addDocument(doc);
+
+      if (random().nextBoolean()) {
+        w.deleteDocuments(new Term("id", ""+random().nextInt(i+1)));
+      }
+    }
+    w.close();
+    d.close();
+  }
 }

Modified: lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/search/TestSort.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/search/TestSort.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/search/TestSort.java (original)
+++ lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/search/TestSort.java Fri Feb 15 22:58:27 2013
@@ -20,11 +20,6 @@ package org.apache.lucene.search;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.BitSet;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Random;
-import java.util.Set;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
@@ -44,7 +39,6 @@ import org.apache.lucene.index.Directory
 import org.apache.lucene.index.FieldInfo.DocValuesType;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.MultiReader;
 import org.apache.lucene.index.RandomIndexWriter;
@@ -53,13 +47,10 @@ import org.apache.lucene.index.StoredDoc
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.FieldValueHitQueue.Entry;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.DocIdBitSet;
-import org.apache.lucene.util.FixedBitSet;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.NamedThreadFactory;
 import org.apache.lucene.util._TestUtil;
@@ -351,11 +342,6 @@ public class TestSort extends LuceneTest
     return getIndex(false, true);
   }
 
-  private IndexSearcher getEmptyIndex()
-  throws IOException {
-    return getIndex(false, false);
-  }
-
   // Set to true if the DV "string" field is indexed as a
   // sorted source:
   private boolean dvStringSorted;
@@ -392,64 +378,6 @@ public class TestSort extends LuceneTest
     super.tearDown();
   }
 
-  // test the sorts by score and document number
-  public void testBuiltInSorts() throws Exception {
-    sort = new Sort();
-    assertMatches(full, queryX, sort, "ACEGI");
-    assertMatches(full, queryY, sort, "BDFHJ");
-
-    sort.setSort(SortField.FIELD_DOC);
-    assertMatches(full, queryX, sort, "ACEGI");
-    assertMatches(full, queryY, sort, "BDFHJ");
-  }
-
-  // test sorts where the type of field is specified
-  public void testTypedSort() throws Exception {
-    sort.setSort(new SortField("int", SortField.Type.INT), SortField.FIELD_DOC);
-    assertMatches(full, queryX, sort, "IGAEC");
-    assertMatches(full, queryY, sort, "DHFJB");
-    
-    sort.setSort(new SortField("float", SortField.Type.FLOAT), SortField.FIELD_DOC);
-    assertMatches(full, queryX, sort, "GCIEA");
-    assertMatches(full, queryY, sort, "DHJFB");
-
-    sort.setSort(new SortField("long", SortField.Type.LONG), SortField.FIELD_DOC);
-    assertMatches(full, queryX, sort, "EACGI");
-    assertMatches(full, queryY, sort, "FBJHD");
-
-    sort.setSort(new SortField("double", SortField.Type.DOUBLE), SortField.FIELD_DOC);
-    assertMatches(full, queryX, sort, "AGICE");
-    assertMatches(full, queryY, sort, "DJHBF");
-    
-    sort.setSort(new SortField("byte", SortField.Type.BYTE), SortField.FIELD_DOC);
-    assertMatches(full, queryX, sort, "CIGAE");
-    assertMatches(full, queryY, sort, "DHFBJ");
-
-    sort.setSort(new SortField("short", SortField.Type.SHORT), SortField.FIELD_DOC);
-    assertMatches(full, queryX, sort, "IAGCE");
-    assertMatches(full, queryY, sort, "DFHBJ");
-
-    sort.setSort(new SortField("string", SortField.Type.STRING), SortField.FIELD_DOC);
-    assertMatches(full, queryX, sort, "AIGEC");
-    assertMatches(full, queryY, sort, "DJHFB");
-    
-    sort.setSort(new SortField("int_dv", SortField.Type.INT), SortField.FIELD_DOC);
-    assertMatches(full, queryX, sort, "IGAEC");
-    assertMatches(full, queryY, sort, "DHFJB");
-
-    sort.setSort(new SortField("float_dv", SortField.Type.FLOAT), SortField.FIELD_DOC);
-    assertMatches(full, queryX, sort, "GCIEA");
-    assertMatches(full, queryY, sort, "DHJFB");
-
-    sort.setSort(new SortField("double_dv", SortField.Type.DOUBLE), SortField.FIELD_DOC);
-    assertMatches(full, queryX, sort, "AGICE");
-    assertMatches(full, queryY, sort, "DJHBF");
-
-    sort.setSort(new SortField("string_dv", getDVStringSortType()), SortField.FIELD_DOC);
-    assertMatches(full, queryX, sort, "AIGEC");
-    assertMatches(full, queryY, sort, "DJHFB");
-  }
-
   private SortField.Type getDVStringSortType() {
     return getDVStringSortType(true);
   }
@@ -609,142 +537,6 @@ public class TestSort extends LuceneTest
     assertFalse("Found sort results out of order", fail);
     searcher.getIndexReader().close();
   }
-  
-  /** 
-   * test sorts where the type of field is specified and a custom field parser 
-   * is used, that uses a simple char encoding. The sorted string contains a 
-   * character beginning from 'A' that is mapped to a numeric value using some 
-   * "funny" algorithm to be different for each data type.
-   */
-  public void testCustomFieldParserSort() throws Exception {
-    // since tests explicilty uses different parsers on the same fieldname
-    // we explicitly check/purge the FieldCache between each assertMatch
-    FieldCache fc = FieldCache.DEFAULT;
-
-
-    sort.setSort(new SortField("parser", new FieldCache.IntParser(){
-      @Override
-      public final int parseInt(final BytesRef term) {
-        return (term.bytes[term.offset]-'A') * 123456;
-      }
-      
-      @Override
-      public TermsEnum termsEnum(Terms terms) throws IOException {
-        return terms.iterator(null);
-      }
-    }), SortField.FIELD_DOC );
-    assertMatches (full, queryA, sort, "JIHGFEDCBA");
-    assertSaneFieldCaches(getTestName() + " IntParser");
-    fc.purgeAllCaches();
-
-    sort.setSort(new SortField("parser", new FieldCache.FloatParser(){
-      @Override
-      public final float parseFloat(final BytesRef term) {
-        return (float) Math.sqrt( term.bytes[term.offset]);
-      }
-      @Override
-      public TermsEnum termsEnum(Terms terms) throws IOException {
-        return terms.iterator(null);
-      }
-    }), SortField.FIELD_DOC );
-    assertMatches (full, queryA, sort, "JIHGFEDCBA");
-    assertSaneFieldCaches(getTestName() + " FloatParser");
-    fc.purgeAllCaches();
-
-    sort.setSort(new SortField("parser", new FieldCache.LongParser(){
-      @Override
-      public final long parseLong(final BytesRef term) {
-        return (term.bytes[term.offset]-'A') * 1234567890L;
-      }
-      
-      @Override
-      public TermsEnum termsEnum(Terms terms) throws IOException {
-        return terms.iterator(null);
-      }
-    }), SortField.FIELD_DOC );
-    assertMatches (full, queryA, sort, "JIHGFEDCBA");
-    assertSaneFieldCaches(getTestName() + " LongParser");
-    fc.purgeAllCaches();
-
-    sort.setSort(new SortField("parser", new FieldCache.DoubleParser(){
-      @Override
-      public final double parseDouble(final BytesRef term) {
-        return Math.pow( term.bytes[term.offset], (term.bytes[term.offset]-'A'));
-      }
-      @Override
-      public TermsEnum termsEnum(Terms terms) throws IOException {
-        return terms.iterator(null);
-      }
-    }), SortField.FIELD_DOC );
-    assertMatches (full, queryA, sort, "JIHGFEDCBA");
-    assertSaneFieldCaches(getTestName() + " DoubleParser");
-    fc.purgeAllCaches();
-
-    sort.setSort(new SortField("parser", new FieldCache.ByteParser(){
-      @Override
-      public final byte parseByte(final BytesRef term) {
-        return (byte) (term.bytes[term.offset]-'A');
-      }
-
-      @Override
-      public TermsEnum termsEnum(Terms terms) throws IOException {
-        return terms.iterator(null);
-      }
-    }), SortField.FIELD_DOC );
-    assertMatches (full, queryA, sort, "JIHGFEDCBA");
-    assertSaneFieldCaches(getTestName() + " ByteParser");
-    fc.purgeAllCaches();
-
-    sort.setSort(new SortField("parser", new FieldCache.ShortParser(){
-      @Override
-      public final short parseShort(final BytesRef term) {
-        return (short) (term.bytes[term.offset]-'A');
-      }
-      @Override
-      public TermsEnum termsEnum(Terms terms) throws IOException {
-        return terms.iterator(null);
-      }
-    }), SortField.FIELD_DOC );
-    assertMatches (full, queryA, sort, "JIHGFEDCBA");
-    assertSaneFieldCaches(getTestName() + " ShortParser");
-    fc.purgeAllCaches();
-  }
-
-  // test sorts when there's nothing in the index
-  public void testEmptyIndex() throws Exception {
-    IndexSearcher empty = getEmptyIndex();
-
-    sort = new Sort();
-    assertMatches(empty, queryX, sort, "");
-
-    sort.setSort(SortField.FIELD_DOC);
-    assertMatches(empty, queryX, sort, "");
-
-    sort.setSort(new SortField("int", SortField.Type.INT), SortField.FIELD_DOC);
-    assertMatches(empty, queryX, sort, "");
-    
-    sort.setSort(new SortField("int_dv", SortField.Type.INT), SortField.FIELD_DOC);
-    assertMatches(empty, queryX, sort, "");
-
-    sort.setSort(new SortField("string", SortField.Type.STRING, true), SortField.FIELD_DOC);
-    assertMatches(empty, queryX, sort, "");
-
-    sort.setSort(new SortField("float", SortField.Type.FLOAT), new SortField("string", SortField.Type.STRING));
-    assertMatches(empty, queryX, sort, "");
-    
-    sort.setSort(new SortField("float_dv", SortField.Type.FLOAT), new SortField("string", SortField.Type.STRING));
-    assertMatches(empty, queryX, sort, "");
-
-    sort.setSort(new SortField("string_dv", getDVStringSortType(false), true), SortField.FIELD_DOC);
-    assertMatches(empty, queryX, sort, "");
-
-    sort.setSort(new SortField("float_dv", SortField.Type.FLOAT),
-                  new SortField("string_dv", getDVStringSortType(false)));
-    assertMatches(empty, queryX, sort, "");
-    
-    sort.setSort(new SortField("float_dv", SortField.Type.FLOAT), new SortField("string_dv", getDVStringSortType(false)));
-    assertMatches(empty, queryX, sort, "");
-  }
 
   static class MyFieldComparator extends FieldComparator<Integer> {
     FieldCache.Ints docValues;
@@ -822,41 +614,6 @@ public class TestSort extends LuceneTest
     assertMatches(full, queryA, sort, "JIHGFEDCBA");
   }
 
-  // test sorts in reverse
-  public void testReverseSort() throws Exception {
-    sort.setSort(new SortField(null, SortField.Type.SCORE, true), SortField.FIELD_DOC);
-    assertMatches(full, queryX, sort, "IEGCA");
-    assertMatches(full, queryY, sort, "JFHDB");
-
-    sort.setSort(new SortField(null, SortField.Type.DOC, true));
-    assertMatches(full, queryX, sort, "IGECA");
-    assertMatches(full, queryY, sort, "JHFDB");
-
-    sort.setSort(new SortField("int", SortField.Type.INT, true));
-    assertMatches(full, queryX, sort, "CAEGI");
-    assertMatches(full, queryY, sort, "BJFHD");
-
-    sort.setSort(new SortField("float", SortField.Type.FLOAT, true));
-    assertMatches(full, queryX, sort, "AECIG");
-    assertMatches(full, queryY, sort, "BFJHD");
-    
-    sort.setSort(new SortField("string", SortField.Type.STRING, true));
-    assertMatches(full, queryX, sort, "CEGIA");
-    assertMatches(full, queryY, sort, "BFHJD");
-    
-    sort.setSort(new SortField("int_dv", SortField.Type.INT, true));
-    assertMatches(full, queryX, sort, "CAEGI");
-    assertMatches(full, queryY, sort, "BJFHD");
-    
-    sort.setSort(new SortField("float_dv", SortField.Type.FLOAT, true));
-    assertMatches(full, queryX, sort, "AECIG");
-    assertMatches(full, queryY, sort, "BFJHD");
-
-    sort.setSort(new SortField("string_dv", getDVStringSortType(), true));
-    assertMatches(full, queryX, sort, "CEGIA");
-    assertMatches(full, queryY, sort, "BFHJD");
-  }
-
   // test sorting when the sort field is empty (undefined) for some of the documents
   public void testEmptyFieldSort() throws Exception {
 
@@ -980,219 +737,6 @@ public class TestSort extends LuceneTest
     assertEquals(docs1.scoreDocs[0].score, docs2.scoreDocs[0].score, 1e-6);
   }
   
-  public void testSortWithoutFillFields() throws Exception {
-    
-    // There was previously a bug in TopFieldCollector when fillFields was set
-    // to false - the same doc and score was set in ScoreDoc[] array. This test
-    // asserts that if fillFields is false, the documents are set properly. It
-    // does not use Searcher's default search methods (with Sort) since all set
-    // fillFields to true.
-    Sort[] sort = new Sort[] { new Sort(SortField.FIELD_DOC), new Sort() };
-    for(int i = 0; i < sort.length; i++) {
-      Query q = new MatchAllDocsQuery();
-      TopDocsCollector<Entry> tdc = TopFieldCollector.create(sort[i], 10, false,
-          false, false, true);
-      
-      full.search(q, tdc);
-      
-      ScoreDoc[] sd = tdc.topDocs().scoreDocs;
-      for(int j = 1; j < sd.length; j++) {
-        assertTrue(sd[j].doc != sd[j - 1].doc);
-      }
-      
-    }
-  }
-
-  public void testSortWithoutScoreTracking() throws Exception {
-
-    // Two Sort criteria to instantiate the multi/single comparators.
-    Sort[] sort = new Sort[] {new Sort(SortField.FIELD_DOC), new Sort() };
-    for(int i = 0; i < sort.length; i++) {
-      Query q = new MatchAllDocsQuery();
-      TopDocsCollector<Entry> tdc = TopFieldCollector.create(sort[i], 10, true, false,
-          false, true);
-      
-      full.search(q, tdc);
-      
-      TopDocs td = tdc.topDocs();
-      ScoreDoc[] sd = td.scoreDocs;
-      for(int j = 0; j < sd.length; j++) {
-        assertTrue(Float.isNaN(sd[j].score));
-      }
-      assertTrue(Float.isNaN(td.getMaxScore()));
-    }
-  }
-  
-  public void testSortWithScoreNoMaxScoreTracking() throws Exception {
-    
-    // Two Sort criteria to instantiate the multi/single comparators.
-    Sort[] sort = new Sort[] {new Sort(SortField.FIELD_DOC), new Sort() };
-    for(int i = 0; i < sort.length; i++) {
-      Query q = new MatchAllDocsQuery();
-      TopDocsCollector<Entry> tdc = TopFieldCollector.create(sort[i], 10, true, true,
-          false, true);
-      
-      full.search(q, tdc);
-      
-      TopDocs td = tdc.topDocs();
-      ScoreDoc[] sd = td.scoreDocs;
-      for(int j = 0; j < sd.length; j++) {
-        assertTrue(!Float.isNaN(sd[j].score));
-      }
-      assertTrue(Float.isNaN(td.getMaxScore()));
-    }
-  }
-  
-  // MultiComparatorScoringNoMaxScoreCollector
-  public void testSortWithScoreNoMaxScoreTrackingMulti() throws Exception {
-    
-    // Two Sort criteria to instantiate the multi/single comparators.
-    Sort[] sort = new Sort[] {new Sort(SortField.FIELD_DOC, SortField.FIELD_SCORE) };
-    for(int i = 0; i < sort.length; i++) {
-      Query q = new MatchAllDocsQuery();
-      TopDocsCollector<Entry> tdc = TopFieldCollector.create(sort[i], 10, true, true,
-          false, true);
-
-      full.search(q, tdc);
-      
-      TopDocs td = tdc.topDocs();
-      ScoreDoc[] sd = td.scoreDocs;
-      for(int j = 0; j < sd.length; j++) {
-        assertTrue(!Float.isNaN(sd[j].score));
-      }
-      assertTrue(Float.isNaN(td.getMaxScore()));
-    }
-  }
-  
-  public void testSortWithScoreAndMaxScoreTracking() throws Exception {
-    
-    // Two Sort criteria to instantiate the multi/single comparators.
-    Sort[] sort = new Sort[] {new Sort(SortField.FIELD_DOC), new Sort() };
-    for(int i = 0; i < sort.length; i++) {
-      Query q = new MatchAllDocsQuery();
-      TopDocsCollector<Entry> tdc = TopFieldCollector.create(sort[i], 10, true, true,
-          true, true);
-      
-      full.search(q, tdc);
-      
-      TopDocs td = tdc.topDocs();
-      ScoreDoc[] sd = td.scoreDocs;
-      for(int j = 0; j < sd.length; j++) {
-        assertTrue(!Float.isNaN(sd[j].score));
-      }
-      assertTrue(!Float.isNaN(td.getMaxScore()));
-    }
-  }
-  
-  public void testOutOfOrderDocsScoringSort() throws Exception {
-
-    // Two Sort criteria to instantiate the multi/single comparators.
-    Sort[] sort = new Sort[] {new Sort(SortField.FIELD_DOC), new Sort() };
-    boolean[][] tfcOptions = new boolean[][] {
-        new boolean[] { false, false, false },
-        new boolean[] { false, false, true },
-        new boolean[] { false, true, false },
-        new boolean[] { false, true, true },
-        new boolean[] { true, false, false },
-        new boolean[] { true, false, true },
-        new boolean[] { true, true, false },
-        new boolean[] { true, true, true },
-    };
-    String[] actualTFCClasses = new String[] {
-        "OutOfOrderOneComparatorNonScoringCollector", 
-        "OutOfOrderOneComparatorScoringMaxScoreCollector", 
-        "OutOfOrderOneComparatorScoringNoMaxScoreCollector", 
-        "OutOfOrderOneComparatorScoringMaxScoreCollector", 
-        "OutOfOrderOneComparatorNonScoringCollector", 
-        "OutOfOrderOneComparatorScoringMaxScoreCollector", 
-        "OutOfOrderOneComparatorScoringNoMaxScoreCollector", 
-        "OutOfOrderOneComparatorScoringMaxScoreCollector" 
-    };
-    
-    BooleanQuery bq = new BooleanQuery();
-    // Add a Query with SHOULD, since bw.scorer() returns BooleanScorer2
-    // which delegates to BS if there are no mandatory clauses.
-    bq.add(new MatchAllDocsQuery(), Occur.SHOULD);
-    // Set minNrShouldMatch to 1 so that BQ will not optimize rewrite to return
-    // the clause instead of BQ.
-    bq.setMinimumNumberShouldMatch(1);
-    for(int i = 0; i < sort.length; i++) {
-      for(int j = 0; j < tfcOptions.length; j++) {
-        TopDocsCollector<Entry> tdc = TopFieldCollector.create(sort[i], 10,
-            tfcOptions[j][0], tfcOptions[j][1], tfcOptions[j][2], false);
-
-        assertTrue(tdc.getClass().getName().endsWith("$"+actualTFCClasses[j]));
-        
-        full.search(bq, tdc);
-        
-        TopDocs td = tdc.topDocs();
-        ScoreDoc[] sd = td.scoreDocs;
-        assertEquals(10, sd.length);
-      }
-    }
-  }
-  
-  // OutOfOrderMulti*Collector
-  public void testOutOfOrderDocsScoringSortMulti() throws Exception {
-
-    // Two Sort criteria to instantiate the multi/single comparators.
-    Sort[] sort = new Sort[] {new Sort(SortField.FIELD_DOC, SortField.FIELD_SCORE) };
-    boolean[][] tfcOptions = new boolean[][] {
-        new boolean[] { false, false, false },
-        new boolean[] { false, false, true },
-        new boolean[] { false, true, false },
-        new boolean[] { false, true, true },
-        new boolean[] { true, false, false },
-        new boolean[] { true, false, true },
-        new boolean[] { true, true, false },
-        new boolean[] { true, true, true },
-    };
-    String[] actualTFCClasses = new String[] {
-        "OutOfOrderMultiComparatorNonScoringCollector", 
-        "OutOfOrderMultiComparatorScoringMaxScoreCollector", 
-        "OutOfOrderMultiComparatorScoringNoMaxScoreCollector", 
-        "OutOfOrderMultiComparatorScoringMaxScoreCollector", 
-        "OutOfOrderMultiComparatorNonScoringCollector", 
-        "OutOfOrderMultiComparatorScoringMaxScoreCollector", 
-        "OutOfOrderMultiComparatorScoringNoMaxScoreCollector", 
-        "OutOfOrderMultiComparatorScoringMaxScoreCollector" 
-    };
-    
-    BooleanQuery bq = new BooleanQuery();
-    // Add a Query with SHOULD, since bw.scorer() returns BooleanScorer2
-    // which delegates to BS if there are no mandatory clauses.
-    bq.add(new MatchAllDocsQuery(), Occur.SHOULD);
-    // Set minNrShouldMatch to 1 so that BQ will not optimize rewrite to return
-    // the clause instead of BQ.
-    bq.setMinimumNumberShouldMatch(1);
-    for(int i = 0; i < sort.length; i++) {
-      for(int j = 0; j < tfcOptions.length; j++) {
-        TopDocsCollector<Entry> tdc = TopFieldCollector.create(sort[i], 10,
-            tfcOptions[j][0], tfcOptions[j][1], tfcOptions[j][2], false);
-
-        assertTrue(tdc.getClass().getName().endsWith("$"+actualTFCClasses[j]));
-        
-        full.search(bq, tdc);
-        
-        TopDocs td = tdc.topDocs();
-        ScoreDoc[] sd = td.scoreDocs;
-        assertEquals(10, sd.length);
-      }
-    }
-  }
-  
-  public void testSortWithScoreAndMaxScoreTrackingNoResults() throws Exception {
-    
-    // Two Sort criteria to instantiate the multi/single comparators.
-    Sort[] sort = new Sort[] {new Sort(SortField.FIELD_DOC), new Sort() };
-    for(int i = 0; i < sort.length; i++) {
-      TopDocsCollector<Entry> tdc = TopFieldCollector.create(sort[i], 10, true, true, true, true);
-      TopDocs td = tdc.topDocs();
-      assertEquals(0, td.totalHits);
-      assertTrue(Float.isNaN(td.getMaxScore()));
-    }
-  }
-  
   // runs a variety of sorts useful for multisearchers
   private void runMultiSorts(IndexSearcher multi, boolean isFull) throws Exception {
     sort.setSort(SortField.FIELD_DOC);
@@ -1331,255 +875,4 @@ public class TestSort extends LuceneTest
     }
     assertEquals(msg, expectedResult, buff.toString());
   }
-
-  public void testEmptyStringVsNullStringSort() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(
-                        TEST_VERSION_CURRENT, new MockAnalyzer(random())));
-    Document doc = new Document();
-    doc.add(newStringField("f", "", Field.Store.NO));
-    doc.add(newStringField("t", "1", Field.Store.NO));
-    w.addDocument(doc);
-    w.commit();
-    doc = new Document();
-    doc.add(newStringField("t", "1", Field.Store.NO));
-    w.addDocument(doc);
-
-    IndexReader r = DirectoryReader.open(w, true);
-    w.close();
-    IndexSearcher s = newSearcher(r);
-    TopDocs hits = s.search(new TermQuery(new Term("t", "1")), null, 10, new Sort(new SortField("f", SortField.Type.STRING)));
-    assertEquals(2, hits.totalHits);
-    // null sorts first
-    assertEquals(1, hits.scoreDocs[0].doc);
-    assertEquals(0, hits.scoreDocs[1].doc);
-    r.close();
-    dir.close();
-  }
-
-  public void testLUCENE2142() throws IOException {
-    Directory indexStore = newDirectory();
-    IndexWriter writer = new IndexWriter(indexStore, newIndexWriterConfig(
-        TEST_VERSION_CURRENT, new MockAnalyzer(random())));
-    for(int i=0; i<5; i++) {
-        Document doc = new Document();
-        doc.add(new StringField("string", "a"+i, Field.Store.NO));
-        doc.add(new StringField("string", "b"+i, Field.Store.NO));
-        writer.addDocument(doc);
-    }
-    writer.forceMerge(1); // enforce one segment to have a higher unique term count in all cases
-    writer.close();
-    sort.setSort(
-        new SortField("string", SortField.Type.STRING),
-        SortField.FIELD_DOC);
-    // this should not throw AIOOBE or RuntimeEx
-    IndexReader reader = DirectoryReader.open(indexStore);
-    IndexSearcher searcher = new IndexSearcher(reader);
-    searcher.search(new MatchAllDocsQuery(), null, 500, sort);
-    reader.close();
-    indexStore.close();
-  }
-
-  public void testCountingCollector() throws Exception {
-    Directory indexStore = newDirectory();
-    RandomIndexWriter writer = new RandomIndexWriter(random(), indexStore);
-    for(int i=0; i<5; i++) {
-      Document doc = new Document();
-      doc.add(new StringField("string", "a"+i, Field.Store.NO));
-      doc.add(new StringField("string", "b"+i, Field.Store.NO));
-      writer.addDocument(doc);
-    }
-    IndexReader reader = writer.getReader();
-    writer.close();
-
-    IndexSearcher searcher = newSearcher(reader);
-    TotalHitCountCollector c = new TotalHitCountCollector();
-    searcher.search(new MatchAllDocsQuery(), null, c);
-    assertEquals(5, c.getTotalHits());
-    reader.close();
-    indexStore.close();
-  }
-
-  private static class RandomFilter extends Filter {
-    private final Random random;
-    private float density;
-    private final List<BytesRef> docValues;
-    public final List<BytesRef> matchValues = Collections.synchronizedList(new ArrayList<BytesRef>());
-
-    // density should be 0.0 ... 1.0
-    public RandomFilter(Random random, float density, List<BytesRef> docValues) {
-      this.random = random;
-      this.density = density;
-      this.docValues = docValues;
-    }
-
-    @Override
-    public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws IOException {
-      final int maxDoc = context.reader().maxDoc();
-      final FieldCache.Ints idSource = FieldCache.DEFAULT.getInts(context.reader(), "id", false);
-      assertNotNull(idSource);
-      final FixedBitSet bits = new FixedBitSet(maxDoc);
-      for(int docID=0;docID<maxDoc;docID++) {
-        if (random.nextFloat() <= density && (acceptDocs == null || acceptDocs.get(docID))) {
-          bits.set(docID);
-          //System.out.println("  acc id=" + idSource.getInt(docID) + " docID=" + docID);
-          matchValues.add(docValues.get(idSource.get(docID)));
-        }
-      }
-
-      return bits;
-    }
-  }
-
-  public void testRandomStringSort() throws Exception {
-    Random random = new Random(random().nextLong());
-
-    final int NUM_DOCS = atLeast(100);
-    final Directory dir = newDirectory();
-    final RandomIndexWriter writer = new RandomIndexWriter(random, dir);
-    final boolean allowDups = random.nextBoolean();
-    final Set<String> seen = new HashSet<String>();
-    final int maxLength = _TestUtil.nextInt(random, 5, 100);
-    if (VERBOSE) {
-      System.out.println("TEST: NUM_DOCS=" + NUM_DOCS + " maxLength=" + maxLength + " allowDups=" + allowDups);
-    }
-
-    int numDocs = 0;
-    final List<BytesRef> docValues = new ArrayList<BytesRef>();
-    // TODO: deletions
-    while (numDocs < NUM_DOCS) {
-      final String s;
-      if (random.nextBoolean()) {
-        s = _TestUtil.randomSimpleString(random, maxLength);
-      } else {
-        s = _TestUtil.randomUnicodeString(random, maxLength);
-      }
-      final BytesRef br = new BytesRef(s);
-
-      if (!allowDups) {
-        if (seen.contains(s)) {
-          continue;
-        }
-        seen.add(s);
-      }
-
-      if (VERBOSE) {
-        System.out.println("  " + numDocs + ": s=" + s);
-      }
-      
-      final Document doc = new Document();
-      doc.add(new SortedDocValuesField("stringdv", br));
-      doc.add(newStringField("string", s, Field.Store.NO));
-      doc.add(new NumericDocValuesField("id", numDocs));
-      docValues.add(br);
-      writer.addDocument(doc);
-      numDocs++;
-
-      if (random.nextInt(40) == 17) {
-        // force flush
-        writer.getReader().close();
-      }
-    }
-
-    final IndexReader r = writer.getReader();
-    writer.close();
-    if (VERBOSE) {
-      System.out.println("  reader=" + r);
-    }
-    
-    final IndexSearcher s = newSearcher(r, false);
-    final int ITERS = atLeast(100);
-    for(int iter=0;iter<ITERS;iter++) {
-      final boolean reverse = random.nextBoolean();
-      final TopFieldDocs hits;
-      final SortField sf;
-      if (random.nextBoolean()) {
-        sf = new SortField("stringdv", SortField.Type.STRING, reverse);
-      } else {
-        sf = new SortField("string", SortField.Type.STRING, reverse);
-      }
-      final Sort sort = new Sort(sf);
-      final int hitCount = _TestUtil.nextInt(random, 1, r.maxDoc() + 20);
-      final RandomFilter f = new RandomFilter(random, random.nextFloat(), docValues);
-      if (random.nextBoolean()) {
-        hits = s.search(new ConstantScoreQuery(f),
-                        hitCount,
-                        sort);
-      } else {
-        hits = s.search(new MatchAllDocsQuery(),
-                        f,
-                        hitCount,
-                        sort);
-      }
-
-      if (VERBOSE) {
-        System.out.println("\nTEST: iter=" + iter + " " + hits.totalHits + " hits; topN=" + hitCount + "; reverse=" + reverse);
-      }
-
-      // Compute expected results:
-      Collections.sort(f.matchValues);
-      if (reverse) {
-        Collections.reverse(f.matchValues);
-      }
-      final List<BytesRef> expected = f.matchValues;
-      if (VERBOSE) {
-        System.out.println("  expected:");
-        for(int idx=0;idx<expected.size();idx++) {
-          System.out.println("    " + idx + ": " + expected.get(idx).utf8ToString());
-          if (idx == hitCount-1) {
-            break;
-          }
-        }
-      }
-      
-      if (VERBOSE) {
-        System.out.println("  actual:");
-        for(int hitIDX=0;hitIDX<hits.scoreDocs.length;hitIDX++) {
-          final FieldDoc fd = (FieldDoc) hits.scoreDocs[hitIDX];
-          System.out.println("    " + hitIDX + ": " + ((BytesRef) fd.fields[0]).utf8ToString());
-        }
-      }
-      for(int hitIDX=0;hitIDX<hits.scoreDocs.length;hitIDX++) {
-        final FieldDoc fd = (FieldDoc) hits.scoreDocs[hitIDX];
-        assertEquals(expected.get(hitIDX), (BytesRef) fd.fields[0]);
-      }
-    }
-
-    r.close();
-    dir.close();
-  }
-
-  public void testMaxScore() throws Exception {
-    Directory d = newDirectory();
-    // Not RIW because we need exactly 2 segs:
-    IndexWriter w = new IndexWriter(d, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
-    int id = 0;
-    for(int seg=0;seg<2;seg++) {
-      for(int docIDX=0;docIDX<10;docIDX++) {
-        Document doc = new Document();
-        doc.add(newStringField("id", ""+docIDX, Field.Store.YES));
-        StringBuilder sb = new StringBuilder();
-        for(int i=0;i<id;i++) {
-          sb.append(' ');
-          sb.append("text");
-        }
-        doc.add(newTextField("body", sb.toString(), Field.Store.NO));
-        w.addDocument(doc);
-        id++;
-      }
-      w.commit();
-    }
-
-    IndexReader r = DirectoryReader.open(w, true);
-    w.close();
-    Query q = new TermQuery(new Term("body", "text"));
-    IndexSearcher s = newSearcher(r);
-    float maxScore = s.search(q , 10).getMaxScore();
-    assertEquals(maxScore, s.search(q, null, 3, Sort.INDEXORDER, random().nextBoolean(), true).getMaxScore(), 0.0);
-    assertEquals(maxScore, s.search(q, null, 3, Sort.RELEVANCE, random().nextBoolean(), true).getMaxScore(), 0.0);
-    assertEquals(maxScore, s.search(q, null, 3, new Sort(new SortField[] {new SortField("id", SortField.Type.INT, false)}), random().nextBoolean(), true).getMaxScore(), 0.0);
-    assertEquals(maxScore, s.search(q, null, 3, new Sort(new SortField[] {new SortField("id", SortField.Type.INT, true)}), random().nextBoolean(), true).getMaxScore(), 0.0);
-    r.close();
-    d.close();
-  }
 }

Modified: lucene/dev/branches/lucene4765/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java (original)
+++ lucene/dev/branches/lucene4765/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java Fri Feb 15 22:58:27 2013
@@ -145,7 +145,9 @@ public class TestDocValuesFieldSources e
 
   public void test() throws IOException {
     for (DocValuesType type : DocValuesType.values()) {
-      test(type);
+      if (type != DocValuesType.SORTED_SET) {
+        test(type);
+      }
     }
   }
 

Modified: lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java (original)
+++ lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java Fri Feb 15 22:58:27 2013
@@ -78,12 +78,14 @@ import java.util.concurrent.ConcurrentHa
 public abstract class PrefixTreeStrategy extends SpatialStrategy {
   protected final SpatialPrefixTree grid;
   private final Map<String, PointPrefixTreeFieldCacheProvider> provider = new ConcurrentHashMap<String, PointPrefixTreeFieldCacheProvider>();
+  protected final boolean simplifyIndexedCells;
   protected int defaultFieldValuesArrayLen = 2;
   protected double distErrPct = SpatialArgs.DEFAULT_DISTERRPCT;// [ 0 TO 0.5 ]
 
-  public PrefixTreeStrategy(SpatialPrefixTree grid, String fieldName) {
+  public PrefixTreeStrategy(SpatialPrefixTree grid, String fieldName, boolean simplifyIndexedCells) {
     super(grid.getSpatialContext(), fieldName);
     this.grid = grid;
+    this.simplifyIndexedCells = simplifyIndexedCells;
   }
 
   /**
@@ -123,7 +125,7 @@ public abstract class PrefixTreeStrategy
 
   public Field[] createIndexableFields(Shape shape, double distErr) {
     int detailLevel = grid.getLevelForDistance(distErr);
-    List<Node> cells = grid.getNodes(shape, detailLevel, true);//true=intermediates cells
+    List<Node> cells = grid.getNodes(shape, detailLevel, true, simplifyIndexedCells);//intermediates cells
 
     //TODO is CellTokenStream supposed to be re-used somehow? see Uwe's comments:
     //  http://code.google.com/p/lucene-spatial-playground/issues/detail?id=4

Modified: lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java (original)
+++ lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java Fri Feb 15 22:58:27 2013
@@ -38,7 +38,8 @@ public class RecursivePrefixTreeStrategy
   private int prefixGridScanLevel;
 
   public RecursivePrefixTreeStrategy(SpatialPrefixTree grid, String fieldName) {
-    super(grid, fieldName);
+    super(grid, fieldName,
+        true);//simplify indexed cells
     prefixGridScanLevel = grid.getMaxLevels() - 4;//TODO this default constant is dependent on the prefix grid size
   }
 

Modified: lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/TermQueryPrefixTreeStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/TermQueryPrefixTreeStrategy.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/TermQueryPrefixTreeStrategy.java (original)
+++ lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/TermQueryPrefixTreeStrategy.java Fri Feb 15 22:58:27 2013
@@ -32,7 +32,7 @@ import java.util.List;
 /**
  * A basic implementation of {@link PrefixTreeStrategy} using a large {@link
  * TermsFilter} of all the nodes from {@link SpatialPrefixTree#getNodes(com.spatial4j.core.shape.Shape,
- * int, boolean)}. It only supports the search of indexed Point shapes.
+ * int, boolean, boolean)}. It only supports the search of indexed Point shapes.
  * <p/>
  * The precision of query shapes (distErrPct) is an important factor in using
  * this Strategy. If the precision is too precise then it will result in many
@@ -43,7 +43,8 @@ import java.util.List;
 public class TermQueryPrefixTreeStrategy extends PrefixTreeStrategy {
 
   public TermQueryPrefixTreeStrategy(SpatialPrefixTree grid, String fieldName) {
-    super(grid, fieldName);
+    super(grid, fieldName,
+        false);//do not simplify indexed cells
   }
 
   @Override
@@ -54,7 +55,9 @@ public class TermQueryPrefixTreeStrategy
 
     Shape shape = args.getShape();
     int detailLevel = grid.getLevelForDistance(args.resolveDistErr(ctx, distErrPct));
-    List<Node> cells = grid.getNodes(shape, detailLevel, false);
+    List<Node> cells = grid.getNodes(shape, detailLevel,
+        false,//no parents
+        true);//simplify
     BytesRef[] terms = new BytesRef[cells.size()];
     int i = 0;
     for (Node cell : cells) {

Modified: lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java (original)
+++ lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java Fri Feb 15 22:58:27 2013
@@ -93,12 +93,6 @@ public class GeohashPrefixTree extends S
     return new GhCell(bytes, offset, len);
   }
 
-  @Override
-  public List<Node> getNodes(Shape shape, int detailLevel, boolean inclParents) {
-    return shape instanceof Point ? super.getNodesAltPoint((Point) shape, detailLevel, inclParents) :
-        super.getNodes(shape, detailLevel, inclParents);
-  }
-
   class GhCell extends Node {
     GhCell(String token) {
       super(token);

Modified: lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java (original)
+++ lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java Fri Feb 15 22:58:27 2013
@@ -156,14 +156,6 @@ public class QuadPrefixTree extends Spat
     return new QuadCell(bytes, offset, len);
   }
 
-  @Override //for performance
-  public List<Node> getNodes(Shape shape, int detailLevel, boolean inclParents) {
-    if (shape instanceof Point)
-      return super.getNodesAltPoint((Point) shape, detailLevel, inclParents);
-    else
-      return super.getNodes(shape, detailLevel, inclParents);
-  }
-
   private void build(
       double x,
       double y,

Modified: lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java (original)
+++ lucene/dev/branches/lucene4765/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java Fri Feb 15 22:58:27 2013
@@ -116,88 +116,86 @@ public abstract class SpatialPrefixTree 
   }
 
   /**
-   * Gets the intersecting & including cells for the specified shape, without exceeding detail level.
-   * The result is a set of cells (no dups), sorted. Unmodifiable.
+   * Gets the intersecting cells for the specified shape, without exceeding
+   * detail level. If a cell is within the query shape then it's marked as a
+   * leaf and none of its children are added.
    * <p/>
-   * This implementation checks if shape is a Point and if so uses an implementation that
-   * recursively calls {@link Node#getSubCell(com.spatial4j.core.shape.Point)}. Cell subclasses
-   * ideally implement that method with a quick implementation, otherwise, subclasses should
-   * override this method to invoke {@link #getNodesAltPoint(com.spatial4j.core.shape.Point, int, boolean)}.
-   * TODO consider another approach returning an iterator -- won't build up all cells in memory.
+   * This implementation checks if shape is a Point and if so returns {@link
+   * #getNodes(com.spatial4j.core.shape.Point, int, boolean)}.
+   *
+   * @param shape       the shape; non-null
+   * @param detailLevel the maximum detail level to get cells for
+   * @param inclParents if true then all parent cells of leaves are returned
+   *                    too. The top world cell is never returned.
+   * @param simplify    for non-point shapes, this will simply/aggregate sets of
+   *                    complete leaves in a cell to its parent, resulting in
+   *                    ~20-25% fewer cells.
+   * @return a set of cells (no dups), sorted, immutable, non-null
    */
-  public List<Node> getNodes(Shape shape, int detailLevel, boolean inclParents) {
+  public List<Node> getNodes(Shape shape, int detailLevel, boolean inclParents,
+                             boolean simplify) {
+    //TODO consider an on-demand iterator -- it won't build up all cells in memory.
     if (detailLevel > maxLevels) {
       throw new IllegalArgumentException("detailLevel > maxLevels");
     }
-
-    List<Node> cells;
     if (shape instanceof Point) {
-      //optimized point algorithm
-      final int initialCapacity = inclParents ? 1 + detailLevel : 1;
-      cells = new ArrayList<Node>(initialCapacity);
-      recursiveGetNodes(getWorldNode(), (Point) shape, detailLevel, true, cells);
-      assert cells.size() == initialCapacity;
-    } else {
-      cells = new ArrayList<Node>(inclParents ? 1024 : 512);
-      recursiveGetNodes(getWorldNode(), shape, detailLevel, inclParents, cells);
-    }
-    if (inclParents) {
-      Node c = cells.remove(0);//remove getWorldNode()
-      assert c.getLevel() == 0;
+      return getNodes((Point) shape, detailLevel, inclParents);
     }
+    List<Node> cells = new ArrayList<Node>(inclParents ? 4096 : 2048);
+    recursiveGetNodes(getWorldNode(), shape, detailLevel, inclParents, simplify, cells);
     return cells;
   }
 
-  private void recursiveGetNodes(Node node, Shape shape, int detailLevel, boolean inclParents,
-                                 Collection<Node> result) {
-    if (node.isLeaf()) {//cell is within shape
-      result.add(node);
-      return;
-    }
-    final Collection<Node> subCells = node.getSubCells(shape);
-    if (node.getLevel() == detailLevel - 1) {
-      if (subCells.size() < node.getSubCellsSize() || node.getLevel() == 0) {
-        if (inclParents)
-          result.add(node);
-        for (Node subCell : subCells) {
-          subCell.setLeaf();
-        }
-        result.addAll(subCells);
-      } else {//a bottom level (i.e. detail level) optimization where all boxes intersect, so use parent cell.
-        node.setLeaf();//the cell may not be strictly within but its close
-        result.add(node);
-      }
-    } else {
-      if (inclParents) {
-        result.add(node);
-      }
-      for (Node subCell : subCells) {
-        recursiveGetNodes(subCell, shape, detailLevel, inclParents, result);//tail call
-      }
+  /**
+   * Returns true if node was added as a leaf. If it wasn't it recursively
+   * descends.
+   */
+  private boolean recursiveGetNodes(Node node, Shape shape, int detailLevel,
+                                    boolean inclParents, boolean simplify,
+                                    List<Node> result) {
+    if (node.getLevel() == detailLevel) {
+      node.setLeaf();//FYI might already be a leaf
     }
-  }
-
-  private void recursiveGetNodes(Node node, Point point, int detailLevel, boolean inclParents,
-                                 Collection<Node> result) {
-    if (inclParents) {
+    if (node.isLeaf()) {
       result.add(node);
+      return true;
     }
-    final Node pCell = node.getSubCell(point);
-    if (node.getLevel() == detailLevel - 1) {
-      pCell.setLeaf();
-      result.add(pCell);
-    } else {
-      recursiveGetNodes(pCell, point, detailLevel, inclParents, result);//tail call
+    if (inclParents && node.getLevel() != 0)
+      result.add(node);
+
+    Collection<Node> subCells = node.getSubCells(shape);
+    int leaves = 0;
+    for (Node subCell : subCells) {
+      if (recursiveGetNodes(subCell, shape, detailLevel, inclParents, simplify, result))
+        leaves++;
+    }
+    //can we simplify?
+    if (simplify && leaves == node.getSubCellsSize() && node.getLevel() != 0) {
+      //Optimization: substitute the parent as a leaf instead of adding all
+      // children as leaves
+
+      //remove the leaves
+      do {
+        result.remove(result.size() - 1);//remove last
+      } while (--leaves > 0);
+      //add node as the leaf
+      node.setLeaf();
+      if (!inclParents) // otherwise it was already added up above
+        result.add(node);
+      return true;
     }
+    return false;
   }
 
   /**
-   * Subclasses might override {@link #getNodes(com.spatial4j.core.shape.Shape, int, boolean)}
-   * and check if the argument is a shape and if so, delegate
-   * to this implementation, which calls {@link #getNode(com.spatial4j.core.shape.Point, int)} and
-   * then calls {@link #getNode(String)} repeatedly if inclParents is true.
+   * A Point-optimized implementation of
+   * {@link #getNodes(com.spatial4j.core.shape.Shape, int, boolean, boolean)}. That
+   * method in facts calls this for points.
+   * <p/>
+   * This implementation depends on {@link #getNode(String)} being fast, as its
+   * called repeatedly when incPlarents is true.
    */
-  protected final List<Node> getNodesAltPoint(Point p, int detailLevel, boolean inclParents) {
+  public List<Node> getNodes(Point p, int detailLevel, boolean inclParents) {
     Node cell = getNode(p, detailLevel);
     if (!inclParents) {
       return Collections.singletonList(cell);

Modified: lucene/dev/branches/lucene4765/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/JtsPolygonTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/JtsPolygonTest.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/JtsPolygonTest.java (original)
+++ lucene/dev/branches/lucene4765/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/JtsPolygonTest.java Fri Feb 15 22:58:27 2013
@@ -18,9 +18,19 @@ package org.apache.lucene.spatial.prefix
  */
 
 import com.spatial4j.core.context.SpatialContextFactory;
+import com.spatial4j.core.shape.Point;
 import com.spatial4j.core.shape.Shape;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.spatial.StrategyTestCase;
 import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree;
+import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree;
+import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
 import org.apache.lucene.spatial.query.SpatialArgs;
 import org.apache.lucene.spatial.query.SpatialOperation;
 import org.junit.Test;
@@ -70,4 +80,39 @@ public class JtsPolygonTest extends Stra
     return args;
   }
 
+  /**
+   * A PrefixTree pruning optimization gone bad.
+   * See <a href="https://issues.apache.org/jira/browse/LUCENE-4770>LUCENE-4770</a>.
+   */
+  @Test
+  public void testBadPrefixTreePrune() throws Exception {
+  
+    Shape area = ctx.readShape("POLYGON((-122.83 48.57, -122.77 48.56, -122.79 48.53, -122.83 48.57))");
+    
+    SpatialPrefixTree trie = new QuadPrefixTree(ctx, 12);
+    TermQueryPrefixTreeStrategy strategy = new TermQueryPrefixTreeStrategy(trie, "geo");
+    Document doc = new Document();
+    doc.add(new TextField("id", "1", Store.YES));
+
+    Field[] fields = strategy.createIndexableFields(area, 0.025);
+    for (Field field : fields) {
+      doc.add(field);  
+    }
+    addDocument(doc);
+
+    Point upperleft = ctx.makePoint(-122.88, 48.54);
+    Point lowerright = ctx.makePoint(-122.82, 48.62);
+    
+    Query query = strategy.makeQuery(new SpatialArgs(SpatialOperation.Intersects, ctx.makeRectangle(upperleft, lowerright)));
+    commit();
+    
+    TopDocs search = indexSearcher.search(query, 10);
+    ScoreDoc[] scoreDocs = search.scoreDocs;
+    for (ScoreDoc scoreDoc : scoreDocs) {
+      System.out.println(indexSearcher.doc(scoreDoc.doc));
+    }
+
+    assertEquals(1, search.totalHits);
+  }
+
 }

Modified: lucene/dev/branches/lucene4765/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/SpatialOpRecursivePrefixTreeTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/SpatialOpRecursivePrefixTreeTest.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/SpatialOpRecursivePrefixTreeTest.java (original)
+++ lucene/dev/branches/lucene4765/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/SpatialOpRecursivePrefixTreeTest.java Fri Feb 15 22:58:27 2013
@@ -113,7 +113,7 @@ public class SpatialOpRecursivePrefixTre
     double distErrPct = ((PrefixTreeStrategy) strategy).getDistErrPct();
     double distErr = SpatialArgs.calcDistanceFromErrPct(snapMe, distErrPct, ctx);
     int detailLevel = grid.getLevelForDistance(distErr);
-    List<Node> cells = grid.getNodes(snapMe, detailLevel, false);
+    List<Node> cells = grid.getNodes(snapMe, detailLevel, false, true);
 
     //calc bounding box of cells.
     double minX = Double.POSITIVE_INFINITY, maxX = Double.NEGATIVE_INFINITY;

Modified: lucene/dev/branches/lucene4765/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTreeTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTreeTest.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTreeTest.java (original)
+++ lucene/dev/branches/lucene4765/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTreeTest.java Fri Feb 15 22:58:27 2013
@@ -18,14 +18,24 @@
 package org.apache.lucene.spatial.prefix.tree;
 
 import com.spatial4j.core.context.SpatialContext;
+import com.spatial4j.core.shape.Point;
 import com.spatial4j.core.shape.Rectangle;
 import com.spatial4j.core.shape.Shape;
-
-import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.spatial.SpatialTestCase;
+import org.apache.lucene.spatial.prefix.TermQueryPrefixTreeStrategy;
+import org.apache.lucene.spatial.query.SpatialArgs;
+import org.apache.lucene.spatial.query.SpatialOperation;
 import org.junit.Before;
 import org.junit.Test;
 
-public class SpatialPrefixTreeTest extends LuceneTestCase {
+public class SpatialPrefixTreeTest extends SpatialTestCase {
 
   //TODO plug in others and test them
   private SpatialContext ctx;
@@ -36,11 +46,12 @@ public class SpatialPrefixTreeTest exten
   public void setUp() throws Exception {
     super.setUp();
     ctx = SpatialContext.GEO;
-    trie = new GeohashPrefixTree(ctx,4);
   }
 
   @Test
   public void testNodeTraverse() {
+    trie = new GeohashPrefixTree(ctx,4);
+
     Node prevN = null;
     Node n = trie.getWorldNode();
     assertEquals(0,n.getLevel());
@@ -57,4 +68,40 @@ public class SpatialPrefixTreeTest exten
       assertTrue(prevNShape.getHeight() > sbox.getHeight());
     }
   }
-}
+  /**
+   * A PrefixTree pruning optimization gone bad.
+   * See <a href="https://issues.apache.org/jira/browse/LUCENE-4770>LUCENE-4770</a>.
+   */
+  @Test
+  public void testBadPrefixTreePrune() throws Exception {
+
+    trie = new QuadPrefixTree(ctx, 12);
+    TermQueryPrefixTreeStrategy strategy = new TermQueryPrefixTreeStrategy(trie, "geo");
+    Document doc = new Document();
+    doc.add(new TextField("id", "1", Store.YES));
+
+    Shape area = ctx.makeRectangle(-122.82, -122.78, 48.54, 48.56);
+
+    Field[] fields = strategy.createIndexableFields(area, 0.025);
+    for (Field field : fields) {
+      doc.add(field);
+    }
+    addDocument(doc);
+
+    Point upperleft = ctx.makePoint(-122.88, 48.54);
+    Point lowerright = ctx.makePoint(-122.82, 48.62);
+
+    Query query = strategy.makeQuery(new SpatialArgs(SpatialOperation.Intersects, ctx.makeRectangle(upperleft, lowerright)));
+
+    commit();
+
+    TopDocs search = indexSearcher.search(query, 10);
+    ScoreDoc[] scoreDocs = search.scoreDocs;
+    for (ScoreDoc scoreDoc : scoreDocs) {
+      System.out.println(indexSearcher.doc(scoreDoc.doc));
+    }
+
+    assertEquals(1, search.totalHits);
+  }
+
+}
\ No newline at end of file

Modified: lucene/dev/branches/lucene4765/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java (original)
+++ lucene/dev/branches/lucene4765/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java Fri Feb 15 22:58:27 2013
@@ -80,7 +80,7 @@ public class AssertingTermVectorsFormat 
   static class AssertingTermVectorsWriter extends TermVectorsWriter {
     private final TermVectorsWriter in;
     private Status docStatus, fieldStatus, termStatus;
-    private int fieldCount, termCount, positionCount;
+    private int docCount, fieldCount, termCount, positionCount;
     boolean hasPositions;
 
     AssertingTermVectorsWriter(TermVectorsWriter in) {
@@ -98,6 +98,7 @@ public class AssertingTermVectorsFormat 
       in.startDocument(numVectorFields);
       docStatus = Status.STARTED;
       fieldCount = numVectorFields;
+      docCount++;
     }
 
     @Override
@@ -167,6 +168,7 @@ public class AssertingTermVectorsFormat 
 
     @Override
     public void finish(FieldInfos fis, int numDocs) throws IOException {
+      assert docCount == numDocs;
       assert docStatus == (numDocs > 0 ? Status.FINISHED : Status.UNDEFINED);
       assert fieldStatus != Status.STARTED;
       assert termStatus != Status.STARTED;
@@ -181,9 +183,6 @@ public class AssertingTermVectorsFormat 
     @Override
     public void close() throws IOException {
       in.close();
-      assert docStatus != Status.STARTED;
-      assert fieldStatus != Status.STARTED;
-      assert termStatus != Status.STARTED;
     }
 
   }

Modified: lucene/dev/branches/lucene4765/lucene/tools/junit4/tests.policy
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/tools/junit4/tests.policy?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/tools/junit4/tests.policy (original)
+++ lucene/dev/branches/lucene4765/lucene/tools/junit4/tests.policy Fri Feb 15 22:58:27 2013
@@ -67,4 +67,7 @@ grant {
   permission java.security.SecurityPermission "getProperty.networkaddress.cache.ttl";
   permission java.security.SecurityPermission "getProperty.networkaddress.cache.negative.ttl";
 
+  // SSL related properties for Solr tests
+  permission java.security.SecurityPermission "getProperty.ssl.*";
+
 };

Modified: lucene/dev/branches/lucene4765/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/solr/CHANGES.txt?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/solr/CHANGES.txt (original)
+++ lucene/dev/branches/lucene4765/solr/CHANGES.txt Fri Feb 15 22:58:27 2013
@@ -41,6 +41,8 @@ Detailed Change List
 Other Changes
 ----------------------
 
+* SOLR-4394: Tests and example configs demonstrating SSL with both server 
+  and client certs (hossman)
 
 ==================  4.2.0 ==================
 
@@ -71,6 +73,10 @@ New Features
 * SOLR-4370: Allow configuring commitWithin to do hard commits. 
   (Mark Miller, Senthuran Sivananthan)
 
+* SOLR-4451: SolrJ, and SolrCloud internals, now use SystemDefaultHttpClient 
+  under the covers -- allowing many HTTP connection related properties to be
+  controlled via 'standard' java system properties.  (hossman)
+
 Bug Fixes
 ----------------------
 
@@ -117,6 +123,8 @@ Bug Fixes
 * SOLR-4426: NRTCachingDirectoryFactory does not initialize maxCachedMB and maxMergeSizeMB
   if <directoryFactory> is not present in solrconfig.xml (Jack Krupansky via shalin)
 
+* SOLR-4463: Fix SolrCoreState reference counting. (Mark Miller)
+
 Optimizations
 ----------------------
 

Modified: lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java (original)
+++ lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java Fri Feb 15 22:58:27 2013
@@ -23,6 +23,9 @@ import java.util.LinkedList;
 import java.util.Random;
 import java.util.concurrent.atomic.AtomicLong;
 
+import java.net.URL;
+import java.net.MalformedURLException;
+
 import javax.servlet.DispatcherType;
 import javax.servlet.Filter;
 import javax.servlet.FilterChain;
@@ -38,8 +41,11 @@ import org.apache.solr.servlet.SolrDispa
 import org.eclipse.jetty.server.Connector;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.bio.SocketConnector;
-import org.eclipse.jetty.server.handler.GzipHandler;
 import org.eclipse.jetty.server.nio.SelectChannelConnector;
+import org.eclipse.jetty.server.ssl.SslConnector;
+import org.eclipse.jetty.server.ssl.SslSocketConnector;
+import org.eclipse.jetty.server.ssl.SslSelectChannelConnector;
+import org.eclipse.jetty.server.handler.GzipHandler;
 import org.eclipse.jetty.server.session.HashSessionIdManager;
 import org.eclipse.jetty.servlet.FilterHolder;
 import org.eclipse.jetty.servlet.ServletContextHandler;
@@ -47,6 +53,7 @@ import org.eclipse.jetty.util.component.
 import org.eclipse.jetty.util.log.Logger;
 import org.eclipse.jetty.util.thread.QueuedThreadPool;
 import org.eclipse.jetty.util.thread.ThreadPool;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
 
 /**
  * Run solr using jetty
@@ -155,22 +162,59 @@ public class JettySolrRunner {
     System.setProperty("solr.solr.home", solrHome);
     if (System.getProperty("jetty.testMode") != null) {
       final String connectorName = System.getProperty("tests.jettyConnector", "SelectChannel");
+
+      // if this property is true, then jetty will be configured to use SSL
+      // leveraging the same system properties as java to specify
+      // the keystore/truststore if they are set
+      //
+      // This means we will use the same truststore, keystore (and keys) for
+      // the server as well as any client actions taken by this JVM in
+      // talking to that server, but for the purposes of testing that should 
+      // be good enough
+      final boolean useSsl = Boolean.getBoolean("tests.jettySsl");
+      final SslContextFactory sslcontext = new SslContextFactory(false);
+
+      if (useSsl) {
+        if (null != System.getProperty("javax.net.ssl.keyStore")) {
+          sslcontext.setKeyStorePath
+            (System.getProperty("javax.net.ssl.keyStore"));
+        }
+        if (null != System.getProperty("javax.net.ssl.keyStorePassword")) {
+          sslcontext.setKeyStorePassword
+            (System.getProperty("javax.net.ssl.keyStorePassword"));
+        }
+        if (null != System.getProperty("javax.net.ssl.trustStore")) {
+          sslcontext.setTrustStore
+            (System.getProperty("javax.net.ssl.trustStore"));
+        }
+        if (null != System.getProperty("javax.net.ssl.trustStorePassword")) {
+          sslcontext.setTrustStorePassword
+            (System.getProperty("javax.net.ssl.trustStorePassword"));
+        }
+        sslcontext.setNeedClientAuth(Boolean.getBoolean("tests.jettySsl.clientAuth"));
+      }
+
       final Connector connector;
       final QueuedThreadPool threadPool;
       if ("SelectChannel".equals(connectorName)) {
-        final SelectChannelConnector c = new SelectChannelConnector();
+        final SelectChannelConnector c = useSsl
+          ? new SslSelectChannelConnector(sslcontext)
+          : new SelectChannelConnector();
         c.setReuseAddress(true);
         c.setLowResourcesMaxIdleTime(1500);
         connector = c;
         threadPool = (QueuedThreadPool) c.getThreadPool();
       } else if ("Socket".equals(connectorName)) {
-        final SocketConnector c = new SocketConnector();
+        final SocketConnector c = useSsl
+          ? new SslSocketConnector(sslcontext)
+          : new SocketConnector();
         c.setReuseAddress(true);
         connector = c;
         threadPool = (QueuedThreadPool) c.getThreadPool();
       } else {
         throw new IllegalArgumentException("Illegal value for system property 'tests.jettyConnector': " + connectorName);
       }
+
       connector.setPort(port);
       connector.setHost("127.0.0.1");
       if (threadPool != null) {
@@ -346,6 +390,31 @@ public class JettySolrRunner {
     return lastPort;
   }
 
+  /**
+   * Returns a base URL consisting of the protocal, host, and port for a 
+   * Connector in use by the Jetty Server contained in this runner.
+   */
+  public URL getBaseUrl() {
+    String protocol = null;
+    try {
+      Connector[] conns = server.getConnectors();
+      if (0 == conns.length) {
+        throw new IllegalStateException("Jetty Server has no Connectors");
+      }
+      Connector c = conns[0];
+      if (c.getLocalPort() < 0) {
+        throw new IllegalStateException("Jetty Connector is not open: " + 
+                                        c.getLocalPort());
+      }
+      protocol = (c instanceof SslConnector) ? "https" : "http";
+      return new URL(protocol, c.getHost(), c.getLocalPort(), context);
+
+    } catch (MalformedURLException e) {
+      throw new  IllegalStateException
+        ("Java could not make sense of protocol: " + protocol, e);
+    }
+  }
+
   public DebugFilter getDebugFilter() {
     return (DebugFilter)debugFilter.getFilter();
   }

Modified: lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java?rev=1446801&r1=1446800&r2=1446801&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java (original)
+++ lucene/dev/branches/lucene4765/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java Fri Feb 15 22:58:27 2013
@@ -185,7 +185,7 @@ public class OverseerCollectionProcessor
     } catch (SolrException ex) {
       SolrException.log(log, "Collection " + operation + " of " + operation
           + " failed");
-      results.add("Operation " + operation + " cause exception:", ex);
+      results.add("Operation " + operation + " caused exception:", ex);
     } finally {
       return new OverseerSolrResponse(results);
     }



Mime
View raw message