lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sim...@apache.org
Subject svn commit: r1402140 [12/17] - in /lucene/dev/branches/LUCENE-2878: ./ dev-tools/ dev-tools/eclipse/ dev-tools/eclipse/dot.settings/ dev-tools/idea/.idea/libraries/ dev-tools/idea/lucene/classification/ dev-tools/maven/ dev-tools/maven/lucene/classific...
Date Thu, 25 Oct 2012 13:10:51 GMT
Modified: lucene/dev/branches/LUCENE-2878/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/regex/RegexQueryCapable.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/regex/RegexQueryCapable.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/regex/RegexQueryCapable.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/regex/RegexQueryCapable.java Thu Oct 25 13:10:25 2012
@@ -23,6 +23,14 @@ package org.apache.lucene.sandbox.querie
  */
 public interface RegexQueryCapable {
   
+  /**
+   * Defines which {@link RegexCapabilities} implementation is used by this instance.
+   * @see #getRegexImplementation()
+   */
   void setRegexImplementation(RegexCapabilities impl);
+  /**
+   * Returns the implementation used by this instance.
+   * @see #setRegexImplementation(RegexCapabilities)
+   */
   RegexCapabilities getRegexImplementation();
 }

Modified: lucene/dev/branches/LUCENE-2878/lucene/site/changes/changes2html.pl
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/site/changes/changes2html.pl?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/site/changes/changes2html.pl (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/site/changes/changes2html.pl Thu Oct 25 13:10:25 2012
@@ -41,17 +41,10 @@ my $first_relid = undef;
 my $second_relid = undef;
 my @releases = ();
 
-my @lines = <>;                        # Get all input at once
+my @lines = <STDIN>;                        # Get all input at once
 
-my $product = '';
-for my $line (@lines) {
-  ($product) = $line =~ /(Solr|Lucene)/i;
-  if ($product) {
-    $product = uc($product);
-    last;
-  }
-}
-my %release_dates = &setup_release_dates;
+my $product = $ARGV[0];
+my %release_dates = &setup_release_dates($ARGV[1]);
 my $in_major_component_versions_section = 0;
 
 
@@ -804,6 +797,7 @@ sub get_release_date {
 #
 sub setup_release_dates {
   my %release_dates;
+  my $file = shift;
   if (uc($product) eq 'LUCENE') {
     %release_dates
        = ( '0.01' => '2000-03-30',      '0.04' => '2000-04-19',
@@ -826,9 +820,7 @@ sub setup_release_dates {
            '3.0.0' => '2009-11-25');
   }
 
-  print STDERR "Retrieving $project_info_url/$product ...\n";
-  my $project_info_json = get_url_contents("$project_info_url/$product");
-
+  my $project_info_json = readFile($file);
   my $project_info = json2perl($project_info_json);
   for my $version (@{$project_info->{versions}}) {
     if ($version->{releaseDate}) {
@@ -844,20 +836,13 @@ sub setup_release_dates {
   return %release_dates;
 }
 
-#
-# returns contents of the passed in url
-#
-sub get_url_contents {
-  my $url = shift;
-  my $tryWget = `wget --no-check-certificate -O - $url`;
-  if ($? eq 0) {
-    return $tryWget;
-  }
-  my $tryCurl = `curl $url`;
-  if ($? eq 0) {
-    return $tryCurl;
-  }
-  die "could not retrieve $url with either wget or curl!";
+sub readFile {
+  my $file = shift;
+  open(F, '<'.$file) || die "could not open $file: $!";
+  local $/ = undef;
+  my $project_info_json = <F>;
+  close(F);
+  return $project_info_json;
 }
 
 #

Modified: lucene/dev/branches/LUCENE-2878/lucene/site/xsl/index.xsl
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/site/xsl/index.xsl?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/site/xsl/index.xsl (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/site/xsl/index.xsl Thu Oct 25 13:10:25 2012
@@ -63,9 +63,10 @@
         <h2>Reference Documents</h2>
           <ul>
             <li><a href="changes/Changes.html">Changes</a>: List of changes in this release.</li>
+            <li><a href="SYSTEM_REQUIREMENTS.html">System Requirements</a>: Minimum and supported Java versions.</li>
             <li><a href="MIGRATE.html">Migration Guide</a>: What changed in Lucene 4; how to migrate code from Lucene 3.x.</li>
             <li><a href="JRE_VERSION_MIGRATION.html">JRE Version Migration</a>: Information about upgrading between major JRE versions.</li>
-            <li><a href="core/org/apache/lucene/codecs/lucene40/package-summary.html#package_description">File Formats</a>: Guide to the supported index format used by Lucene.  This can be customized by using <a href="core/org/apache/lucene/codecs/package-summary.html#package_description">an alternate codec</a>.</li>
+            <li><a href="core/org/apache/lucene/codecs/lucene41/package-summary.html#package_description">File Formats</a>: Guide to the supported index format used by Lucene.  This can be customized by using <a href="core/org/apache/lucene/codecs/package-summary.html#package_description">an alternate codec</a>.</li>
             <li><a href="core/org/apache/lucene/search/package-summary.html#package_description">Search and Scoring in Lucene</a>: Introduction to how Lucene scores documents.</li>
             <li><a href="core/org/apache/lucene/search/similarities/TFIDFSimilarity.html">Classic Scoring Formula</a>: Formula of Lucene's classic <a href="http://en.wikipedia.org/wiki/Vector_Space_Model">Vector Space</a> implementation. (look <a href="core/org/apache/lucene/search/similarities/package-summary.html#package_description">here</a> for other models)</li>
             <li><a href="queryparser/org/apache/lucene/queryparser/classic/package-summary.html#package_description">Classic QueryParser Syntax</a>: Overview of the Classic QueryParser's syntax and features.</li>

Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/SpatialStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/SpatialStrategy.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/SpatialStrategy.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/SpatialStrategy.java Thu Oct 25 13:10:25 2012
@@ -40,12 +40,12 @@ import org.apache.lucene.spatial.query.S
  *   <li>What types of query shapes can be used?</li>
  *   <li>What types of query operations are supported?
  *   This might vary per shape.</li>
- *   <li>Are there caches?  Under what circumstances are they used?
- *   Roughly how big are they?  Is it segmented by Lucene segments, such as is
- *   done by the Lucene {@link org.apache.lucene.search.FieldCache} and
- *   {@link org.apache.lucene.index.DocValues} (ideal) or is it for the entire
- *   index?
+ *   <li>Does it use the {@link org.apache.lucene.search.FieldCache}, {@link
+ *   org.apache.lucene.index.DocValues} or some other type of cache?  When?
  * </ul>
+ * If a strategy only supports certain shapes at index or query time, then in
+ * general it will throw an exception if given an incompatible one.  It will not
+ * be coerced into compatibility.
  * <p/>
  * Note that a SpatialStrategy is not involved with the Lucene stored field
  * values of shapes, which is immaterial to indexing & search.
@@ -85,7 +85,7 @@ public abstract class SpatialStrategy {
   }
 
   /**
-   * Returns the IndexableField(s) from the <code>shape</code> that are to be
+   * Returns the IndexableField(s) from the {@code shape} that are to be
    * added to the {@link org.apache.lucene.document.Document}.  These fields
    * are expected to be marked as indexed and not stored.
    * <p/>
@@ -96,6 +96,7 @@ public abstract class SpatialStrategy {
    * since it doesn't use it.
    *
    * @return Not null nor will it have null elements.
+   * @throws UnsupportedOperationException if given a shape incompatible with the strategy
    */
   public abstract Field[] createIndexableFields(Shape shape);
 
@@ -111,6 +112,10 @@ public abstract class SpatialStrategy {
    * and {@link Shape} from the supplied {@code args}.
    * The default implementation is
    * <pre>return new ConstantScoreQuery(makeFilter(args));</pre>
+   *
+   * @throws UnsupportedOperationException If the strategy does not support the shape in {@code args}
+   * @throws org.apache.lucene.spatial.query.UnsupportedSpatialOperation If the strategy does not support the {@link
+   * org.apache.lucene.spatial.query.SpatialOperation} in {@code args}.
    */
   public ConstantScoreQuery makeQuery(SpatialArgs args) {
     return new ConstantScoreQuery(makeFilter(args));
@@ -124,13 +129,17 @@ public abstract class SpatialStrategy {
    * {@link #makeQuery(org.apache.lucene.spatial.query.SpatialArgs)}
    * then this method could be simply:
    * <pre>return new QueryWrapperFilter(makeQuery(args).getQuery());</pre>
+   *
+   * @throws UnsupportedOperationException If the strategy does not support the shape in {@code args}
+   * @throws org.apache.lucene.spatial.query.UnsupportedSpatialOperation If the strategy does not support the {@link
+   * org.apache.lucene.spatial.query.SpatialOperation} in {@code args}.
    */
   public abstract Filter makeFilter(SpatialArgs args);
 
   /**
    * Returns a ValueSource with values ranging from 1 to 0, depending inversely
    * on the distance from {@link #makeDistanceValueSource(com.spatial4j.core.shape.Point)}.
-   * The formula is <code>c/(d + c)</code> where 'd' is the distance and 'c' is
+   * The formula is {@code c/(d + c)} where 'd' is the distance and 'c' is
    * one tenth the distance to the farthest edge from the center. Thus the
    * scores will be 1 for indexed points at the center of the query shape and as
    * low as ~0.1 at its furthest edges.

Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java Thu Oct 25 13:10:25 2012
@@ -43,8 +43,28 @@ import org.apache.lucene.spatial.query.U
 
 
 /**
- * Based on GeoPortal's
- * <a href="http://geoportal.svn.sourceforge.net/svnroot/geoportal/Geoportal/trunk/src/com/esri/gpt/catalog/lucene/SpatialClauseAdapter.java">SpatialClauseAdapter</a>.
+ * A SpatialStrategy for indexing and searching Rectangles by storing its
+ * coordinates in numeric fields. It supports all {@link SpatialOperation}s and
+ * has a custom overlap relevancy. It is based on GeoPortal's <a
+ * href="http://geoportal.svn.sourceforge.net/svnroot/geoportal/Geoportal/trunk/src/com/esri/gpt/catalog/lucene/SpatialClauseAdapter.java">SpatialClauseAdapter</a>.
+ *
+ * <h4>Characteristics:</h4>
+ * <ul>
+ * <li>Only indexes Rectangles; just one per field value.</li>
+ * <li>Can query only by a Rectangle.</li>
+ * <li>Supports all {@link SpatialOperation}s.</li>
+ * <li>Uses the FieldCache for any sorting / relevancy.</li>
+ * </ul>
+ *
+ * <h4>Implementation:</h4>
+ * This uses 4 double fields for minX, maxX, minY, maxY
+ * and a boolean to mark a dateline cross. Depending on the particular {@link
+ * SpatialOperation}s, there is a variety of {@link NumericRangeQuery}s to be
+ * done.
+ * The {@link #makeBBoxAreaSimilarityValueSource(com.spatial4j.core.shape.Rectangle)}
+ * works by calculating the query bbox overlap percentage against the indexed
+ * shape overlap percentage. The indexed shape's coordinates are retrieved from
+ * the {@link org.apache.lucene.search.FieldCache}.
  *
  * @lucene.experimental
  */
@@ -95,7 +115,7 @@ public class BBoxStrategy extends Spatia
   public Field[] createIndexableFields(Shape shape) {
     if (shape instanceof Rectangle)
       return createIndexableFields((Rectangle)shape);
-    throw new IllegalArgumentException("Can only index Rectangle, not " + shape);
+    throw new UnsupportedOperationException("Can only index Rectangle, not " + shape);
   }
 
   public Field[] createIndexableFields(Rectangle bbox) {
@@ -150,7 +170,7 @@ public class BBoxStrategy extends Spatia
   private Query makeSpatialQuery(SpatialArgs args) {
     Shape shape = args.getShape();
     if (!(shape instanceof Rectangle))
-      throw new IllegalArgumentException("Can only query by Rectangle, not " + shape);
+      throw new UnsupportedOperationException("Can only query by Rectangle, not " + shape);
 
     Rectangle bbox = (Rectangle) shape;
     Query spatial = null;

Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PointPrefixTreeFieldCacheProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PointPrefixTreeFieldCacheProvider.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PointPrefixTreeFieldCacheProvider.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PointPrefixTreeFieldCacheProvider.java Thu Oct 25 13:10:25 2012
@@ -40,8 +40,7 @@ public class PointPrefixTreeFieldCachePr
     this.grid = grid;
   }
 
-  //A kluge that this is a field
-  private Node scanCell = null;
+  private Node scanCell = null;//re-used in readShape to save GC
 
   @Override
   protected Point readShape(BytesRef term) {

Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java Thu Oct 25 13:10:25 2012
@@ -37,8 +37,41 @@ import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
 /**
- * Abstract SpatialStrategy which provides common functionality for those 
- * Strategys which use {@link SpatialPrefixTree}s
+ * An abstract SpatialStrategy based on {@link SpatialPrefixTree}. The two
+ * subclasses are {@link RecursivePrefixTreeStrategy} and {@link
+ * TermQueryPrefixTreeStrategy}.  This strategy is most effective as a fast
+ * approximate spatial search filter.
+ *
+ * <h4>Characteristics:</h4>
+ * <ul>
+ * <li>Can index any shape; however only {@link RecursivePrefixTreeStrategy}
+ * can effectively search non-point shapes. <em>Not tested.</em></li>
+ * <li>Can index a variable number of shapes per field value. This strategy
+ * can do it via multiple calls to {@link #createIndexableFields(com.spatial4j.core.shape.Shape)}
+ * for a document or by giving it some sort of Shape aggregate (e.g. JTS
+ * WKT MultiPoint).  The shape's boundary is approximated to a grid precision.
+ * </li>
+ * <li>Can query with any shape.  The shape's boundary is approximated to a grid
+ * precision.</li>
+ * <li>Only {@link org.apache.lucene.spatial.query.SpatialOperation#Intersects}
+ * is supported.  If only points are indexed then this is effectively equivalent
+ * to IsWithin.</li>
+ * <li>The strategy supports {@link #makeDistanceValueSource(com.spatial4j.core.shape.Point)}
+ * even for multi-valued data.  However, <em>it will likely be removed in the
+ * future</em> in lieu of using another strategy with a more scalable
+ * implementation.  Use of this call is the only
+ * circumstance in which a cache is used.  The cache is simple but as such
+ * it doesn't scale to large numbers of points nor is it real-time-search
+ * friendly.</li>
+ * </ul>
+ *
+ * <h4>Implementation:</h4>
+ * The {@link SpatialPrefixTree} does most of the work, for example returning
+ * a list of terms representing grids of various sizes for a supplied shape.
+ * An important
+ * configuration item is {@link #setDistErrPct(double)} which balances
+ * shape precision against scalability.  See those javadocs.
+ *
  * @lucene.internal
  */
 public abstract class PrefixTreeStrategy extends SpatialStrategy {
@@ -52,7 +85,12 @@ public abstract class PrefixTreeStrategy
     this.grid = grid;
   }
 
-  /** Used in the in-memory ValueSource as a default ArrayList length for this field's array of values, per doc. */
+  /**
+   * A memory hint used by {@link #makeDistanceValueSource(com.spatial4j.core.shape.Point)}
+   * for how big the initial size of each Document's array should be. The
+   * default is 2.  Set this to slightly more than the default expected number
+   * of points per document.
+   */
   public void setDefaultFieldValuesArrayLen(int defaultFieldValuesArrayLen) {
     this.defaultFieldValuesArrayLen = defaultFieldValuesArrayLen;
   }
@@ -62,8 +100,14 @@ public abstract class PrefixTreeStrategy
   }
 
   /**
-   * The default measure of shape precision affecting indexed and query shapes.
-   * Specific shapes at index and query time can use something different.
+   * The default measure of shape precision affecting shapes at index and query
+   * times. Points don't use this as they are always indexed at the configured
+   * maximum precision ({@link org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree#getMaxLevels()});
+   * this applies to all other shapes. Specific shapes at index and query time
+   * can use something different than this default value.  If you don't set a
+   * default then the default is {@link SpatialArgs#DEFAULT_DISTERRPCT} --
+   * 2.5%.
+   *
    * @see org.apache.lucene.spatial.query.SpatialArgs#getDistErrPct()
    */
   public void setDistErrPct(double distErrPct) {
@@ -81,7 +125,8 @@ public abstract class PrefixTreeStrategy
     List<Node> cells = grid.getNodes(shape, detailLevel, true);//true=intermediates cells
     //If shape isn't a point, add a full-resolution center-point so that
     // PointPrefixTreeFieldCacheProvider has the center-points.
-    // TODO index each center of a multi-point? Yes/no?
+    //TODO index each point of a multi-point or other aggregate.
+    //TODO remove this once support for a distance ValueSource is removed.
     if (!(shape instanceof Point)) {
       Point ctr = shape.getCenter();
       //TODO should be smarter; don't index 2 tokens for this in CellTokenStream. Harmless though.

Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeFilter.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeFilter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeFilter.java Thu Oct 25 13:10:25 2012
@@ -34,11 +34,15 @@ import java.io.IOException;
 import java.util.LinkedList;
 
 /**
- * Performs a spatial intersection filter against a field indexed with {@link SpatialPrefixTree}, a Trie.
- * SPT yields terms (grids) at length 1 and at greater lengths corresponding to greater precisions.
- * This filter recursively traverses each grid length and uses methods on {@link Shape} to efficiently know
- * that all points at a prefix fit in the shape or not to either short-circuit unnecessary traversals or to efficiently
- * load all enclosed points.
+ * Performs a spatial intersection filter between a query shape and a field
+ * indexed with {@link SpatialPrefixTree}, a Trie. SPT yields terms (grids) at
+ * length 1 (aka "Level 1") and at greater lengths corresponding to greater
+ * precisions. This filter recursively traverses each grid length and uses
+ * methods on {@link Shape} to efficiently know that all points at a prefix fit
+ * in the shape or not to either short-circuit unnecessary traversals or to
+ * efficiently load all enclosed points.  If no indexed data lies in a portion
+ * of the shape then that portion of the query shape is quickly passed over
+ * without decomposing the shape unnecessarily.
  *
  * @lucene.internal
  */
@@ -165,7 +169,7 @@ RE "scan" threshold:
 
   @Override
   public String toString() {
-    return "GeoFilter{fieldName='" + fieldName + '\'' + ", shape=" + queryShape + '}';
+    return getClass().getSimpleName()+"{fieldName='" + fieldName + '\'' + ", shape=" + queryShape + '}';
   }
 
   @Override

Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java Thu Oct 25 13:10:25 2012
@@ -25,7 +25,11 @@ import org.apache.lucene.spatial.query.S
 import org.apache.lucene.spatial.query.UnsupportedSpatialOperation;
 
 /**
- * Based on {@link RecursivePrefixTreeFilter}.
+ * A {@link PrefixTreeStrategy} which uses {@link RecursivePrefixTreeFilter}.
+ * This strategy has support for searching non-point shapes (note: not tested).
+ * Even a query shape with distErrPct=0 (fully precise to the grid) should have
+ * good performance for typical data, unless there is a lot of indexed data
+ * coincident with the shape's edge.
  *
  * @lucene.experimental
  */
@@ -38,6 +42,13 @@ public class RecursivePrefixTreeStrategy
     prefixGridScanLevel = grid.getMaxLevels() - 4;//TODO this default constant is dependent on the prefix grid size
   }
 
+  /**
+   * Sets the grid level [1-maxLevels] at which indexed terms are scanned brute-force
+   * instead of by grid decomposition.  By default this is maxLevels - 4.  The
+   * final level, maxLevels, is always scanned.
+   *
+   * @param prefixGridScanLevel 1 to maxLevels
+   */
   public void setPrefixGridScanLevel(int prefixGridScanLevel) {
     //TODO if negative then subtract from maxlevels
     this.prefixGridScanLevel = prefixGridScanLevel;

Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/TermQueryPrefixTreeStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/TermQueryPrefixTreeStrategy.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/TermQueryPrefixTreeStrategy.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/TermQueryPrefixTreeStrategy.java Thu Oct 25 13:10:25 2012
@@ -30,8 +30,13 @@ import org.apache.lucene.spatial.query.U
 import java.util.List;
 
 /**
- * A basic implementation using a large {@link TermsFilter} of all the nodes from
- * {@link SpatialPrefixTree#getNodes(com.spatial4j.core.shape.Shape, int, boolean)}.
+ * A basic implementation of {@link PrefixTreeStrategy} using a large {@link
+ * TermsFilter} of all the nodes from {@link SpatialPrefixTree#getNodes(com.spatial4j.core.shape.Shape,
+ * int, boolean)}. It only supports the search of indexed Point shapes.
+ * <p/>
+ * The precision of query shapes (distErrPct) is an important factor in using
+ * this Strategy. If the precision is too precise then it will result in many
+ * terms which will amount to a slower query.
  *
  * @lucene.experimental
  */

Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java Thu Oct 25 13:10:25 2012
@@ -29,7 +29,9 @@ import java.util.List;
 
 
 /**
- * A SpatialPrefixGrid based on Geohashes.  Uses {@link GeohashUtils} to do all the geohash work.
+ * A {@link SpatialPrefixTree} based on
+ * <a href="http://en.wikipedia.org/wiki/Geohash">Geohashes</a>.
+ * Uses {@link GeohashUtils} to do all the geohash work.
  *
  * @lucene.experimental
  */

Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/Node.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/Node.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/Node.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/Node.java Thu Oct 25 13:10:25 2012
@@ -27,7 +27,7 @@ import java.util.Collections;
 import java.util.List;
 
 /**
- * Represents a grid cell. These are not necessarily threadsafe, although new Cell("") (world cell) must be.
+ * Represents a grid cell. These are not necessarily thread-safe, although new Cell("") (world cell) must be.
  *
  * @lucene.experimental
  */

Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java Thu Oct 25 13:10:25 2012
@@ -31,8 +31,9 @@ import java.util.List;
 import java.util.Locale;
 
 /**
- * Implementation of {@link SpatialPrefixTree} which uses a quad tree
- * (http://en.wikipedia.org/wiki/Quadtree)
+ * A {@link SpatialPrefixTree} which uses a
+ * <a href="http://en.wikipedia.org/wiki/Quadtree">quad tree</a> in which an
+ * indexed term will be generated for each node, 'A', 'B', 'C', 'D'.
  *
  * @lucene.experimental
  */

Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java Thu Oct 25 13:10:25 2012
@@ -28,10 +28,13 @@ import java.util.Collections;
 import java.util.List;
 
 /**
- * A spatial Prefix Tree, or Trie, which decomposes shapes into prefixed strings at variable lengths corresponding to
- * variable precision.  Each string corresponds to a spatial region.
- *
- * Implementations of this class should be thread-safe and immutable once initialized.
+ * A spatial Prefix Tree, or Trie, which decomposes shapes into prefixed strings
+ * at variable lengths corresponding to variable precision.   Each string
+ * corresponds to a rectangular spatial region.  This approach is
+ * also referred to "Grids", "Tiles", and "Spatial Tiers".
+ * <p/>
+ * Implementations of this class should be thread-safe and immutable once
+ * initialized.
  *
  * @lucene.experimental
  */

Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTreeFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTreeFactory.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTreeFactory.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTreeFactory.java Thu Oct 25 13:10:25 2012
@@ -31,6 +31,9 @@ import java.util.Map;
 public abstract class SpatialPrefixTreeFactory {
 
   private static final double DEFAULT_GEO_MAX_DETAIL_KM = 0.001;//1m
+  public static final String PREFIX_TREE = "prefixTree";
+  public static final String MAX_LEVELS = "maxLevels";
+  public static final String MAX_DIST_ERR = "maxDistErr";
 
   protected Map<String, String> args;
   protected SpatialContext ctx;
@@ -42,7 +45,7 @@ public abstract class SpatialPrefixTreeF
    */
   public static SpatialPrefixTree makeSPT(Map<String,String> args, ClassLoader classLoader, SpatialContext ctx) {
     SpatialPrefixTreeFactory instance;
-    String cname = args.get("prefixTree");
+    String cname = args.get(PREFIX_TREE);
     if (cname == null)
       cname = ctx.isGeo() ? "geohash" : "quad";
     if ("geohash".equalsIgnoreCase(cname))
@@ -68,14 +71,14 @@ public abstract class SpatialPrefixTreeF
   }
 
   protected void initMaxLevels() {
-    String mlStr = args.get("maxLevels");
+    String mlStr = args.get(MAX_LEVELS);
     if (mlStr != null) {
       maxLevels = Integer.valueOf(mlStr);
       return;
     }
 
     double degrees;
-    String maxDetailDistStr = args.get("maxDetailDist");
+    String maxDetailDistStr = args.get(MAX_DIST_ERR);
     if (maxDetailDistStr == null) {
       if (!ctx.isGeo()) {
         return;//let default to max
@@ -83,9 +86,6 @@ public abstract class SpatialPrefixTreeF
       degrees = DistanceUtils.dist2Degrees(DEFAULT_GEO_MAX_DETAIL_KM, DistanceUtils.EARTH_MEAN_RADIUS_KM);
     } else {
       degrees = Double.parseDouble(maxDetailDistStr);
-      if (ctx.isGeo()) {
-        degrees = DistanceUtils.dist2Degrees(Double.parseDouble(maxDetailDistStr), DistanceUtils.EARTH_MEAN_RADIUS_KM);
-      }
     }
     maxLevels = getLevelForDistance(degrees);
   }

Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialArgs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialArgs.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialArgs.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialArgs.java Thu Oct 25 13:10:25 2012
@@ -24,6 +24,7 @@ import com.spatial4j.core.shape.Shape;
 
 /**
  * Principally holds the query {@link Shape} and the {@link SpatialOperation}.
+ * It's used as an argument to some methods on {@link org.apache.lucene.spatial.SpatialStrategy}.
  *
  * @lucene.experimental
  */
@@ -72,7 +73,6 @@ public class SpatialArgs {
    * Gets the error distance that specifies how precise the query shape is. This
    * looks at {@link #getDistErr()}, {@link #getDistErrPct()}, and {@code
    * defaultDistErrPct}.
-   * @param ctx
    * @param defaultDistErrPct 0 to 0.5
    * @return >= 0
    */

Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialArgsParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialArgsParser.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialArgsParser.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialArgsParser.java Thu Oct 25 13:10:25 2012
@@ -34,7 +34,7 @@ import java.util.StringTokenizer;
  * ShapeReadWriter#readShape(String)}. There are some optional name-value pair parameters
  * that follow the closing parenthesis.  Example:
  * <pre>
- *   Intersects(-10,20,-8,22) distPec=0.025
+ *   Intersects(-10,20,-8,22) distErrPct=0.025
  * </pre>
  * <p/>
  * In the future it would be good to support something at least semi-standardized like a
@@ -86,7 +86,7 @@ public class SpatialArgsParser {
       throw new IllegalArgumentException("missing body : " + v, null);
     }
 
-    Shape shape = new ShapeReadWriter(ctx).readShape(body);
+    Shape shape = ctx.readShape(body);
     SpatialArgs args = new SpatialArgs(op, shape);
 
     if (v.length() > (edx + 1)) {

Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/query/UnsupportedSpatialOperation.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/query/UnsupportedSpatialOperation.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/query/UnsupportedSpatialOperation.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/query/UnsupportedSpatialOperation.java Thu Oct 25 13:10:25 2012
@@ -18,7 +18,7 @@ package org.apache.lucene.spatial.query;
  */
 
 /**
- * Exception thrown when the strategy cannot implement the requested operation.
+ * Exception thrown when the {@link org.apache.lucene.spatial.SpatialStrategy} cannot implement the requested operation.
  * @lucene.experimental
  */
 public class UnsupportedSpatialOperation extends UnsupportedOperationException {

Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/util/ShapeFieldCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/util/ShapeFieldCache.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/util/ShapeFieldCache.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/util/ShapeFieldCache.java Thu Oct 25 13:10:25 2012
@@ -24,7 +24,10 @@ import java.util.List;
 
 /**
  * Bounded Cache of Shapes associated with docIds.  Note, multiple Shapes can be
- * associated with a given docId
+ * associated with a given docId.
+ * <p>
+ * WARNING: This class holds the data in an extremely inefficient manner as all Points are in memory as objects and they
+ * are stored in many ArrayLists (one per document).  So it works but doesn't scale.  It will be replaced in the future.
  *
  * @lucene.internal
  */

Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/util/ValueSourceFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/util/ValueSourceFilter.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/util/ValueSourceFilter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/util/ValueSourceFilter.java Thu Oct 25 13:10:25 2012
@@ -28,11 +28,12 @@ import org.apache.lucene.util.Bits;
 import java.io.IOException;
 
 /**
- * Filter that matches all documents where a valuesource is
+ * Filter that matches all documents where a ValueSource is
  * in between a range of <code>min</code> and <code>max</code> inclusive.
  * @lucene.internal
  */
 public class ValueSourceFilter extends Filter {
+  //TODO see https://issues.apache.org/jira/browse/LUCENE-4251  (move out of spatial & improve)
 
   final Filter startingFilter;
   final ValueSource source;

Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/vector/DistanceValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/vector/DistanceValueSource.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/vector/DistanceValueSource.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/java/org/apache/lucene/spatial/vector/DistanceValueSource.java Thu Oct 25 13:10:25 2012
@@ -30,19 +30,20 @@ import java.io.IOException;
 import java.util.Map;
 
 /**
- * An implementation of the Lucene ValueSource model that returns the distance.
+ * An implementation of the Lucene ValueSource model that returns the distance
+ * for a {@link PointVectorStrategy}.
  *
  * @lucene.internal
  */
 public class DistanceValueSource extends ValueSource {
 
-  private TwoDoublesStrategy strategy;
+  private PointVectorStrategy strategy;
   private final Point from;
 
   /**
    * Constructor.
    */
-  public DistanceValueSource(TwoDoublesStrategy strategy, Point from) {
+  public DistanceValueSource(PointVectorStrategy strategy, Point from) {
     this.strategy = strategy;
     this.from = from;
   }

Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/test/org/apache/lucene/spatial/DistanceStrategyTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/test/org/apache/lucene/spatial/DistanceStrategyTest.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/test/org/apache/lucene/spatial/DistanceStrategyTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/test/org/apache/lucene/spatial/DistanceStrategyTest.java Thu Oct 25 13:10:25 2012
@@ -29,7 +29,7 @@ import org.apache.lucene.spatial.prefix.
 import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree;
 import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree;
 import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
-import org.apache.lucene.spatial.vector.TwoDoublesStrategy;
+import org.apache.lucene.spatial.vector.PointVectorStrategy;
 import org.junit.Test;
 
 import java.io.IOException;
@@ -54,7 +54,7 @@ public class DistanceStrategyTest extend
     strategy = new TermQueryPrefixTreeStrategy(grid, "termquery_geohash");
     ctorArgs.add(new Object[]{new Param(strategy)});
 
-    strategy = new TwoDoublesStrategy(ctx, "twodoubles");
+    strategy = new PointVectorStrategy(ctx, "pointvector");
     ctorArgs.add(new Object[]{new Param(strategy)});
 
     strategy = new BBoxStrategy(ctx, "bbox");

Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/test/org/apache/lucene/spatial/PortedSolr3Test.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/test/org/apache/lucene/spatial/PortedSolr3Test.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/test/org/apache/lucene/spatial/PortedSolr3Test.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/test/org/apache/lucene/spatial/PortedSolr3Test.java Thu Oct 25 13:10:25 2012
@@ -21,7 +21,6 @@ import com.carrotsearch.randomizedtestin
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import com.spatial4j.core.context.SpatialContext;
 import com.spatial4j.core.distance.DistanceUtils;
-import com.spatial4j.core.io.ShapeReadWriter;
 import com.spatial4j.core.shape.Point;
 import com.spatial4j.core.shape.Shape;
 import org.apache.lucene.search.FilteredQuery;
@@ -34,7 +33,7 @@ import org.apache.lucene.spatial.prefix.
 import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
 import org.apache.lucene.spatial.query.SpatialArgs;
 import org.apache.lucene.spatial.query.SpatialOperation;
-import org.apache.lucene.spatial.vector.TwoDoublesStrategy;
+import org.apache.lucene.spatial.vector.PointVectorStrategy;
 import org.junit.Test;
 
 import java.io.IOException;
@@ -68,7 +67,7 @@ public class PortedSolr3Test extends Str
     strategy = new TermQueryPrefixTreeStrategy(grid, "termquery_geohash");
     ctorArgs.add(new Object[]{new Param(strategy)});
 
-    strategy = new TwoDoublesStrategy(ctx, "twodoubles");
+    strategy = new PointVectorStrategy(ctx, "pointvector");
     ctorArgs.add(new Object[]{new Param(strategy)});
 
     return ctorArgs;
@@ -162,7 +161,7 @@ public class PortedSolr3Test extends Str
 
   private void _checkHits(boolean bbox, String ptStr, double distKM, int assertNumFound, int... assertIds) {
     SpatialOperation op = SpatialOperation.Intersects;
-    Point pt = (Point) new ShapeReadWriter(ctx).readShape(ptStr);
+    Point pt = (Point) ctx.readShape(ptStr);
     double distDEG = DistanceUtils.dist2Degrees(distKM, DistanceUtils.EARTH_MEAN_RADIUS_KM);
     Shape shape = ctx.makeCircle(pt, distDEG);
     if (bbox)

Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialExample.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialExample.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialExample.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialExample.java Thu Oct 25 13:10:25 2012
@@ -19,7 +19,6 @@ package org.apache.lucene.spatial;
 
 import com.spatial4j.core.context.SpatialContext;
 import com.spatial4j.core.distance.DistanceUtils;
-import com.spatial4j.core.io.ShapeReadWriter;
 import com.spatial4j.core.shape.Point;
 import com.spatial4j.core.shape.Shape;
 import org.apache.lucene.document.Document;
@@ -111,7 +110,7 @@ public class SpatialExample extends Luce
     //When parsing a string to a shape, the presence of a comma means it's y-x
     // order (lon, lat)
     indexWriter.addDocument(newSampleDocument(
-        4, new ShapeReadWriter(ctx).readShape("-50.7693246, 60.9289094")));
+        4, ctx.readShape("-50.7693246, 60.9289094")));
 
     indexWriter.addDocument(newSampleDocument(
         20, ctx.makePoint(0.1,0.1), ctx.makePoint(0, 0)));

Modified: lucene/dev/branches/LUCENE-2878/lucene/spatial/src/test/org/apache/lucene/spatial/StrategyTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/spatial/src/test/org/apache/lucene/spatial/StrategyTestCase.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/spatial/src/test/org/apache/lucene/spatial/StrategyTestCase.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/spatial/src/test/org/apache/lucene/spatial/StrategyTestCase.java Thu Oct 25 13:10:25 2012
@@ -19,7 +19,6 @@ package org.apache.lucene.spatial;
  */
 
 import com.spatial4j.core.context.SpatialContext;
-import com.spatial4j.core.io.ShapeReadWriter;
 import com.spatial4j.core.io.sample.SampleData;
 import com.spatial4j.core.io.sample.SampleDataReader;
 import com.spatial4j.core.shape.Shape;
@@ -35,6 +34,7 @@ import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.spatial.query.SpatialArgsParser;
 import org.junit.Assert;
 
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
@@ -89,7 +89,7 @@ public abstract class StrategyTestCase e
       Document document = new Document();
       document.add(new StringField("id", data.id, Field.Store.YES));
       document.add(new StringField("name", data.name, Field.Store.YES));
-      Shape shape = new ShapeReadWriter(ctx).readShape(data.shape);
+      Shape shape = ctx.readShape(data.shape);
       shape = convertShapeFromGetDocuments(shape);
       if (shape != null) {
         for (Field f : strategy.createIndexableFields(shape)) {
@@ -110,8 +110,11 @@ public abstract class StrategyTestCase e
   }
 
   protected Iterator<SampleData> getSampleData(String testDataFile) throws IOException {
-    return new SampleDataReader(
-        getClass().getClassLoader().getResourceAsStream("data/"+testDataFile) );
+    String path = "data/" + testDataFile;
+    InputStream stream = getClass().getClassLoader().getResourceAsStream(path);
+    if (stream == null)
+      throw new FileNotFoundException("classpath resource not found: "+path);
+    return new SampleDataReader(stream);
   }
 
   protected Iterator<SpatialTestQuery> getTestQueries(String testQueryFile, SpatialContext ctx) throws IOException {
@@ -130,7 +133,7 @@ public abstract class StrategyTestCase e
       SearchResults got = executeQuery(strategy.makeQuery(q.args), 100);
       if (storeShape && got.numFound > 0) {
         //check stored value is there & parses
-        assertNotNull(new ShapeReadWriter(ctx).readShape(got.results.get(0).document.get(strategy.getFieldName())));
+        assertNotNull(ctx.readShape(got.results.get(0).document.get(strategy.getFieldName())));
       }
       if (concern.orderIsImportant) {
         Iterator<String> ids = q.ids.iterator();
@@ -174,7 +177,7 @@ public abstract class StrategyTestCase e
   }
 
   protected void adoc(String id, String shapeStr) throws IOException {
-    Shape shape = shapeStr==null ? null : new ShapeReadWriter(ctx).readShape(shapeStr);
+    Shape shape = shapeStr==null ? null : ctx.readShape(shapeStr);
     addDocument(newDoc(id, shape));
   }
   protected void adoc(String id, Shape shape) throws IOException {

Modified: lucene/dev/branches/LUCENE-2878/lucene/suggest/src/java/org/apache/lucene/search/spell/DirectSpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/suggest/src/java/org/apache/lucene/search/spell/DirectSpellChecker.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/suggest/src/java/org/apache/lucene/search/spell/DirectSpellChecker.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/suggest/src/java/org/apache/lucene/search/spell/DirectSpellChecker.java Thu Oct 25 13:10:25 2012
@@ -313,7 +313,7 @@ public class DirectSpellChecker {
    * @param suggestMode specifies when to return suggested words
    * @param accuracy return only suggested words that match with this similarity
    * @return sorted list of the suggested words according to the comparator
-   * @throws IOException
+   * @throws IOException If there is a low-level I/O error.
    */
   public SuggestWord[] suggestSimilar(Term term, int numSug, IndexReader ir, 
       SuggestMode suggestMode, float accuracy) throws IOException {

Modified: lucene/dev/branches/LUCENE-2878/lucene/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java Thu Oct 25 13:10:25 2012
@@ -449,7 +449,7 @@ public class SpellChecker implements jav
 
   /**
    * Removes all terms from the spell check index.
-   * @throws IOException
+   * @throws IOException If there is a low-level I/O error.
    * @throws AlreadyClosedException if the Spellchecker is already closed
    */
   public void clearIndex() throws IOException {
@@ -467,8 +467,8 @@ public class SpellChecker implements jav
 
   /**
    * Check whether the word exists in the index.
-   * @param word
-   * @throws IOException
+   * @param word word to check
+   * @throws IOException If there is a low-level I/O error.
    * @throws AlreadyClosedException if the Spellchecker is already closed
    * @return true if the word exists in the index
    */
@@ -490,7 +490,7 @@ public class SpellChecker implements jav
    * @param config {@link IndexWriterConfig} to use
    * @param fullMerge whether or not the spellcheck index should be fully merged
    * @throws AlreadyClosedException if the Spellchecker is already closed
-   * @throws IOException
+   * @throws IOException If there is a low-level I/O error.
    */
   public final void indexDictionary(Dictionary dict, IndexWriterConfig config, boolean fullMerge) throws IOException {
     synchronized (modifyCurrentIndexLock) {

Modified: lucene/dev/branches/LUCENE-2878/lucene/suggest/src/java/org/apache/lucene/search/spell/WordBreakSpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/suggest/src/java/org/apache/lucene/search/spell/WordBreakSpellChecker.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/suggest/src/java/org/apache/lucene/search/spell/WordBreakSpellChecker.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/suggest/src/java/org/apache/lucene/search/spell/WordBreakSpellChecker.java Thu Oct 25 13:10:25 2012
@@ -82,16 +82,13 @@ public class WordBreakSpellChecker {
    * lower score is generally preferred over a higher score.
    * </p>
    * 
-   * @param term
-   * @param maxSuggestions
-   * @param ir
    * @param suggestMode
    *          - default = {@link SuggestMode#SUGGEST_WHEN_NOT_IN_INDEX}
    * @param sortMethod
    *          - default =
    *          {@link BreakSuggestionSortMethod#NUM_CHANGES_THEN_MAX_FREQUENCY}
    * @return one or more arrays of words formed by breaking up the original term
-   * @throws IOException
+   * @throws IOException If there is a low-level I/O error.
    */
   public SuggestWord[][] suggestWordBreaks(Term term, int maxSuggestions,
       IndexReader ir, SuggestMode suggestMode,
@@ -159,12 +156,8 @@ public class WordBreakSpellChecker {
    * included term.
    * </p>
    * 
-   * @param terms
-   * @param maxSuggestions
-   * @param ir
-   * @param suggestMode
    * @return an array of words generated by combining original terms
-   * @throws IOException
+   * @throws IOException If there is a low-level I/O error.
    */
   public CombineSuggestion[] suggestWordCombinations(Term[] terms,
       int maxSuggestions, IndexReader ir, SuggestMode suggestMode)

Modified: lucene/dev/branches/LUCENE-2878/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java Thu Oct 25 13:10:25 2012
@@ -56,7 +56,6 @@ import org.apache.lucene.util.fst.Util.M
  * Input weights must be between 0 and {@link Integer#MAX_VALUE}, any
  * other values will be rejected.
  * 
- * @see Util#shortestPaths(FST, FST.Arc, Comparator, int)
  * @lucene.experimental
  */
 public class WFSTCompletionLookup extends Lookup {
@@ -144,6 +143,11 @@ public class WFSTCompletionLookup extend
   @Override
   public List<LookupResult> lookup(CharSequence key, boolean onlyMorePopular, int num) {
     assert num > 0;
+
+    if (onlyMorePopular) {
+      throw new IllegalArgumentException("this suggester only works with onlyMorePopular=false");
+    }
+
     BytesRef scratch = new BytesRef(key);
     int prefixLength = scratch.length;
     Arc<Long> arc = new Arc<Long>();
@@ -168,12 +172,14 @@ public class WFSTCompletionLookup extend
         return results; // that was quick
       }
     }
-    
+
     // complete top-N
     MinResult<Long> completions[] = null;
     try {
-      completions = Util.shortestPaths(fst, arc, weightComparator, num);
-    } catch (IOException bogus) { throw new RuntimeException(bogus); }
+      completions = Util.shortestPaths(fst, arc, prefixOutput, weightComparator, num, !exactFirst);
+    } catch (IOException bogus) {
+      throw new RuntimeException(bogus);
+    }
     
     BytesRef suffix = new BytesRef(8);
     for (MinResult<Long> completion : completions) {
@@ -183,7 +189,7 @@ public class WFSTCompletionLookup extend
       scratch.append(suffix);
       spare.grow(scratch.length);
       UnicodeUtil.UTF8toUTF16(scratch, spare);
-      results.add(new LookupResult(spare.toString(), decodeWeight(prefixOutput + completion.output)));
+      results.add(new LookupResult(spare.toString(), decodeWeight(completion.output)));
     }
     return results;
   }

Modified: lucene/dev/branches/LUCENE-2878/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java Thu Oct 25 13:10:25 2012
@@ -19,6 +19,7 @@ package org.apache.lucene.search.suggest
 
 import java.io.BufferedReader;
 import java.io.InputStreamReader;
+import java.lang.reflect.Constructor;
 import java.net.URL;
 import java.nio.charset.Charset;
 import java.util.ArrayList;
@@ -30,7 +31,11 @@ import java.util.Random;
 import java.util.concurrent.Callable;
 
 import org.apache.lucene.util.*;
-import org.apache.lucene.search.suggest.Lookup;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.search.suggest.Lookup; // javadocs
+import org.apache.lucene.search.suggest.analyzing.AnalyzingSuggester;
 import org.apache.lucene.search.suggest.fst.FSTCompletionLookup;
 import org.apache.lucene.search.suggest.fst.WFSTCompletionLookup;
 import org.apache.lucene.search.suggest.jaspell.JaspellLookup;
@@ -49,7 +54,8 @@ public class LookupBenchmarkTest extends
       JaspellLookup.class, 
       TSTLookup.class,
       FSTCompletionLookup.class,
-      WFSTCompletionLookup.class);
+      WFSTCompletionLookup.class,
+      AnalyzingSuggester.class);
 
   private final static int rounds = 15;
   private final static int warmup = 5;
@@ -133,10 +139,19 @@ public class LookupBenchmarkTest extends
     System.err.println("-- RAM consumption");
     for (Class<? extends Lookup> cls : benchmarkClasses) {
       Lookup lookup = buildLookup(cls, dictionaryInput);
+      long sizeInBytes;
+      if (lookup instanceof AnalyzingSuggester) {
+        // Just get size of FST: else we are also measuring
+        // size of MockAnalyzer which is non-trivial and
+        // varies depending on test seed:
+        sizeInBytes = ((AnalyzingSuggester) lookup).sizeInBytes();
+      } else {
+        sizeInBytes = RamUsageEstimator.sizeOf(lookup);
+      }
       System.err.println(
           String.format(Locale.ROOT, "%-15s size[B]:%,13d",
               lookup.getClass().getSimpleName(), 
-              RamUsageEstimator.sizeOf(lookup)));
+              sizeInBytes));
     }
   }
 
@@ -144,7 +159,13 @@ public class LookupBenchmarkTest extends
    * Create {@link Lookup} instance and populate it. 
    */
   private Lookup buildLookup(Class<? extends Lookup> cls, TermFreq[] input) throws Exception {
-    Lookup lookup = cls.newInstance();
+    Lookup lookup = null;
+    try {
+      lookup = cls.newInstance();
+    } catch (InstantiationException e) {
+      Constructor<? extends Lookup> ctor = cls.getConstructor(Analyzer.class);
+      lookup = ctor.newInstance(new MockAnalyzer(random, MockTokenizer.KEYWORD, false));
+    }
     lookup.build(new TermFreqArrayIterator(input));
     return lookup;
   }

Modified: lucene/dev/branches/LUCENE-2878/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java Thu Oct 25 13:10:25 2012
@@ -44,7 +44,13 @@ public class WFSTCompletionTest extends 
     assertEquals(1, results.size());
     assertEquals("foo", results.get(0).key.toString());
     assertEquals(50, results.get(0).value, 0.01F);
-    
+
+    // make sure we don't get a dup exact suggestion:
+    results = suggester.lookup(_TestUtil.stringToCharSequence("foo", random), false, 2);
+    assertEquals(1, results.size());
+    assertEquals("foo", results.get(0).key.toString());
+    assertEquals(50, results.get(0).value, 0.01F);
+
     // top N of 1 for 'bar': we return this even though barbar is higher
     results = suggester.lookup(_TestUtil.stringToCharSequence("bar", random), false, 1);
     assertEquals(1, results.size());
@@ -69,6 +75,54 @@ public class WFSTCompletionTest extends 
     assertEquals("barbara", results.get(2).key.toString());
     assertEquals(6, results.get(2).value, 0.01F);
   }
+
+  public void testExactFirst() throws Exception {
+
+    WFSTCompletionLookup suggester = new WFSTCompletionLookup(true);
+
+    suggester.build(new TermFreqArrayIterator(new TermFreq[] {
+          new TermFreq("x y", 20),
+          new TermFreq("x", 2),
+        }));
+
+    for(int topN=1;topN<4;topN++) {
+      List<LookupResult> results = suggester.lookup("x", false, topN);
+
+      assertEquals(Math.min(topN, 2), results.size());
+
+      assertEquals("x", results.get(0).key);
+      assertEquals(2, results.get(0).value);
+
+      if (topN > 1) {
+        assertEquals("x y", results.get(1).key);
+        assertEquals(20, results.get(1).value);
+      }
+    }
+  }
+
+  public void testNonExactFirst() throws Exception {
+
+    WFSTCompletionLookup suggester = new WFSTCompletionLookup(false);
+
+    suggester.build(new TermFreqArrayIterator(new TermFreq[] {
+          new TermFreq("x y", 20),
+          new TermFreq("x", 2),
+        }));
+
+    for(int topN=1;topN<4;topN++) {
+      List<LookupResult> results = suggester.lookup("x", false, topN);
+
+      assertEquals(Math.min(topN, 2), results.size());
+
+      assertEquals("x y", results.get(0).key);
+      assertEquals(20, results.get(0).value);
+
+      if (topN > 1) {
+        assertEquals("x", results.get(1).key);
+        assertEquals(2, results.get(1).value);
+      }
+    }
+  }
   
   public void testRandom() throws Exception {
     int numWords = atLeast(1000);

Modified: lucene/dev/branches/LUCENE-2878/lucene/test-framework/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/test-framework/build.xml?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/test-framework/build.xml (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/test-framework/build.xml Thu Oct 25 13:10:25 2012
@@ -32,19 +32,13 @@
   </path>
 
   <!-- 
-      Specialize compile-core to depend on lucene-core compilation,
-      and *not* to depend on clover; clover already includes the
-      test-framework sources in each module's test instrumentation.
+      Specialize compile-core to depend on lucene-core and lucene-codecs compilation.
    -->
-  <target name="compile-core" depends="init,compile-lucene-core,compile-codecs"
-          description="Compiles test-framework classes">
-    <compile srcdir="${src.dir}" destdir="${build.dir}/classes/java">
-      <classpath refid="classpath"/>
-    </compile>
-    <copy todir="${build.dir}/classes/java">
-      <fileset dir="${src.dir}/../resources" erroronmissingdir="no"/>
-    </copy>
-  </target>
+  <target name="compile-core" depends="init,compile-lucene-core,compile-codecs,common.compile-core"
+          description="Compiles test-framework classes"/>
+
+  <!-- redefine the clover setup, because we dont want to run clover for the test-framework -->
+  <target name="-clover.setup" if="run.clover"/>
 
   <target name="javadocs-core" depends="javadocs"/>
   <target name="javadocs" depends="init,javadocs-lucene-core,javadocs-lucene-codecs">

Modified: lucene/dev/branches/LUCENE-2878/lucene/test-framework/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/test-framework/ivy.xml?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/test-framework/ivy.xml (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/test-framework/ivy.xml Thu Oct 25 13:10:25 2012
@@ -32,8 +32,8 @@
       <dependency org="org.apache.ant" name="ant" rev="1.8.2" transitive="false" />
 
       <dependency org="junit" name="junit" rev="4.10" transitive="false" conf="default->*;junit4-stdalone->*" />
-      <dependency org="com.carrotsearch.randomizedtesting" name="junit4-ant" rev="2.0.1" transitive="false" conf="default->*;junit4-stdalone->*" />
-      <dependency org="com.carrotsearch.randomizedtesting" name="randomizedtesting-runner" rev="2.0.1" transitive="false" conf="default->*;junit4-stdalone->*" />
+      <dependency org="com.carrotsearch.randomizedtesting" name="junit4-ant" rev="2.0.3" transitive="false" conf="default->*;junit4-stdalone->*" />
+      <dependency org="com.carrotsearch.randomizedtesting" name="randomizedtesting-runner" rev="2.0.3" transitive="false" conf="default->*;junit4-stdalone->*" />
 
       <exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/> 
     </dependencies>

Modified: lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingCodec.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingCodec.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingCodec.java Thu Oct 25 13:10:25 2012
@@ -17,14 +17,13 @@ package org.apache.lucene.codecs.asserti
  * limitations under the License.
  */
 
-import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.FilterCodec;
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.codecs.TermVectorsFormat;
-import org.apache.lucene.codecs.lucene40.Lucene40Codec;
+import org.apache.lucene.codecs.lucene41.Lucene41Codec;
 
 /**
- * Acts like {@link Lucene40Codec} but with additional asserts.
+ * Acts like {@link Lucene41Codec} but with additional asserts.
  */
 public final class AssertingCodec extends FilterCodec {
 
@@ -32,12 +31,7 @@ public final class AssertingCodec extend
   private final TermVectorsFormat vectors = new AssertingTermVectorsFormat();
 
   public AssertingCodec() {
-    super("Asserting");
-  }
-
-  @Override
-  protected Codec delegate() {
-    return Codec.forName("Lucene40");
+    super("Asserting", new Lucene41Codec());
   }
 
   @Override

Modified: lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java Thu Oct 25 13:10:25 2012
@@ -27,7 +27,7 @@ import org.apache.lucene.codecs.Postings
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.codecs.TermStats;
 import org.apache.lucene.codecs.TermsConsumer;
-import org.apache.lucene.codecs.lucene40.Lucene40PostingsFormat;
+import org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat;
 import org.apache.lucene.index.AssertingAtomicReader;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfo.IndexOptions;
@@ -38,10 +38,10 @@ import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.OpenBitSet;
 
 /**
- * Just like {@link Lucene40PostingsFormat} but with additional asserts.
+ * Just like {@link Lucene41PostingsFormat} but with additional asserts.
  */
 public final class AssertingPostingsFormat extends PostingsFormat {
-  private final PostingsFormat in = new Lucene40PostingsFormat();
+  private final PostingsFormat in = new Lucene41PostingsFormat();
   
   public AssertingPostingsFormat() {
     super("Asserting");

Modified: lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java Thu Oct 25 13:10:25 2012
@@ -38,8 +38,8 @@ import org.apache.lucene.codecs.blockter
 import org.apache.lucene.codecs.blockterms.TermsIndexWriterBase;
 import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexReader;
 import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexWriter;
-import org.apache.lucene.codecs.lucene40.Lucene40PostingsReader;
-import org.apache.lucene.codecs.lucene40.Lucene40PostingsWriter;
+import org.apache.lucene.codecs.lucene41.Lucene41PostingsReader;
+import org.apache.lucene.codecs.lucene41.Lucene41PostingsWriter;
 import org.apache.lucene.codecs.mockintblock.MockFixedIntBlockPostingsFormat;
 import org.apache.lucene.codecs.mockintblock.MockVariableIntBlockPostingsFormat;
 import org.apache.lucene.codecs.mocksep.MockSingleIntFactory;
@@ -133,9 +133,17 @@ public final class MockRandomPostingsFor
 
   @Override
   public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
+    int minSkipInterval;
+    if (state.segmentInfo.getDocCount() > 1000000) {
+      // Test2BPostings can OOME otherwise:
+      minSkipInterval = 3;
+    } else {
+      minSkipInterval = 2;
+    }
+
     // we pull this before the seed intentionally: because its not consumed at runtime
     // (the skipInterval is written into postings header)
-    int skipInterval = _TestUtil.nextInt(seedRandom, 2, 10);
+    int skipInterval = _TestUtil.nextInt(seedRandom, minSkipInterval, 10);
     
     if (LuceneTestCase.VERBOSE) {
       System.out.println("MockRandomCodec: skipInterval=" + skipInterval);
@@ -166,7 +174,8 @@ public final class MockRandomPostingsFor
       if (LuceneTestCase.VERBOSE) {
         System.out.println("MockRandomCodec: writing Standard postings");
       }
-      postingsWriter = new Lucene40PostingsWriter(state, skipInterval);
+      // TODO: randomize variables like acceptibleOverHead?!
+      postingsWriter = new Lucene41PostingsWriter(state, skipInterval);
     }
 
     if (random.nextBoolean()) {
@@ -305,7 +314,7 @@ public final class MockRandomPostingsFor
       if (LuceneTestCase.VERBOSE) {
         System.out.println("MockRandomCodec: reading Standard postings");
       }
-      postingsReader = new Lucene40PostingsReader(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
+      postingsReader = new Lucene41PostingsReader(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
     }
 
     if (random.nextBoolean()) {

Modified: lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/codecs/nestedpulsing/NestedPulsingPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/codecs/nestedpulsing/NestedPulsingPostingsFormat.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/codecs/nestedpulsing/NestedPulsingPostingsFormat.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/codecs/nestedpulsing/NestedPulsingPostingsFormat.java Thu Oct 25 13:10:25 2012
@@ -26,8 +26,8 @@ import org.apache.lucene.codecs.FieldsPr
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.codecs.PostingsReaderBase;
 import org.apache.lucene.codecs.PostingsWriterBase;
-import org.apache.lucene.codecs.lucene40.Lucene40PostingsReader;
-import org.apache.lucene.codecs.lucene40.Lucene40PostingsWriter;
+import org.apache.lucene.codecs.lucene41.Lucene41PostingsReader;
+import org.apache.lucene.codecs.lucene41.Lucene41PostingsWriter;
 import org.apache.lucene.codecs.pulsing.PulsingPostingsReader;
 import org.apache.lucene.codecs.pulsing.PulsingPostingsWriter;
 import org.apache.lucene.index.SegmentReadState;
@@ -35,7 +35,7 @@ import org.apache.lucene.index.SegmentWr
 import org.apache.lucene.util.IOUtils;
 
 /**
- * Pulsing(1, Pulsing(2, Lucene40))
+ * Pulsing(1, Pulsing(2, Lucene41))
  * @lucene.experimental
  */
 // TODO: if we create PulsingPostingsBaseFormat then we
@@ -55,7 +55,7 @@ public final class NestedPulsingPostings
     // Terms dict
     boolean success = false;
     try {
-      docsWriter = new Lucene40PostingsWriter(state);
+      docsWriter = new Lucene41PostingsWriter(state);
 
       pulsingWriterInner = new PulsingPostingsWriter(2, docsWriter);
       pulsingWriter = new PulsingPostingsWriter(1, pulsingWriterInner);
@@ -77,7 +77,7 @@ public final class NestedPulsingPostings
     PostingsReaderBase pulsingReader = null;
     boolean success = false;
     try {
-      docsReader = new Lucene40PostingsReader(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
+      docsReader = new Lucene41PostingsReader(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
       pulsingReaderInner = new PulsingPostingsReader(docsReader);
       pulsingReader = new PulsingPostingsReader(pulsingReaderInner);
       FieldsProducer ret = new BlockTreeTermsReader(

Modified: lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/AlcoholicMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/AlcoholicMergePolicy.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/AlcoholicMergePolicy.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/AlcoholicMergePolicy.java Thu Oct 25 13:10:25 2012
@@ -47,6 +47,7 @@ public class AlcoholicMergePolicy extend
   
   public AlcoholicMergePolicy(TimeZone tz, Random random) {
     this.calendar = new GregorianCalendar(tz, Locale.ROOT);
+    calendar.setTimeInMillis(_TestUtil.nextLong(random, 0, Long.MAX_VALUE));
     this.random = random;
     maxMergeSize = _TestUtil.nextInt(random, 1024*1024, Integer.MAX_VALUE);
   }

Modified: lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java Thu Oct 25 13:10:25 2012
@@ -261,6 +261,9 @@ public class AssertingAtomicReader exten
       assert state != DocsEnumState.START : "freq() called before nextDoc()/advance()";
       assert state != DocsEnumState.FINISHED : "freq() called after NO_MORE_DOCS";
       int freq = super.freq();
+      if (freq == 0) {
+        System.out.println();
+      }
       assert freq > 0;
       return freq;
     }

Modified: lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/DocHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/DocHelper.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/DocHelper.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/DocHelper.java Thu Oct 25 13:10:25 2012
@@ -257,9 +257,6 @@ class DocHelper {
    * Writes the document to the directory using a segment
    * named "test"; returns the SegmentInfo describing the new
    * segment 
-   * @param dir
-   * @param doc
-   * @throws IOException
    */ 
   public static SegmentInfoPerCommit writeDoc(Random random, Directory dir, Document doc) throws IOException
   {
@@ -270,11 +267,6 @@ class DocHelper {
    * Writes the document to the directory using the analyzer
    * and the similarity score; returns the SegmentInfo
    * describing the new segment
-   * @param dir
-   * @param analyzer
-   * @param similarity
-   * @param doc
-   * @throws IOException
    */ 
   public static SegmentInfoPerCommit writeDoc(Random random, Directory dir, Analyzer analyzer, Similarity similarity, Document doc) throws IOException {
     IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig( /* LuceneTestCase.newIndexWriterConfig(random, */ 

Modified: lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java Thu Oct 25 13:10:25 2012
@@ -72,8 +72,8 @@ public final class FieldFilterAtomicRead
   public void document(final int docID, final StoredFieldVisitor visitor) throws IOException {
     super.document(docID, new StoredFieldVisitor() {
       @Override
-      public void binaryField(FieldInfo fieldInfo, byte[] value, int offset, int length) throws IOException {
-        visitor.binaryField(fieldInfo, value, offset, length);
+      public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException {
+        visitor.binaryField(fieldInfo, value);
       }
 
       @Override

Modified: lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/MockRandomMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/MockRandomMergePolicy.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/MockRandomMergePolicy.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/MockRandomMergePolicy.java Thu Oct 25 13:10:25 2012
@@ -19,11 +19,13 @@ package org.apache.lucene.index;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
-import java.util.Random;
 import java.util.Map;
+import java.util.Random;
 
+import org.apache.lucene.index.MergePolicy.MergeTrigger;
 import org.apache.lucene.util._TestUtil;
 
 /**
@@ -39,18 +41,30 @@ public class MockRandomMergePolicy exten
   }
 
   @Override
-  public MergeSpecification findMerges(SegmentInfos segmentInfos) {
+  public MergeSpecification findMerges(MergeTrigger mergeTrigger, SegmentInfos segmentInfos) {
     MergeSpecification mergeSpec = null;
     //System.out.println("MRMP: findMerges sis=" + segmentInfos);
 
-    if (segmentInfos.size() > 1 && random.nextInt(5) == 3) {
-      
-      List<SegmentInfoPerCommit> segments = new ArrayList<SegmentInfoPerCommit>(segmentInfos.asList());
+    int numSegments = segmentInfos.size();
+
+    List<SegmentInfoPerCommit> segments = new ArrayList<SegmentInfoPerCommit>();
+    final Collection<SegmentInfoPerCommit> merging = writer.get().getMergingSegments();
+
+    for(SegmentInfoPerCommit sipc : segmentInfos) {
+      if (!merging.contains(sipc)) {
+        segments.add(sipc);
+      }
+    }
+
+    numSegments = segments.size();
+
+    if (numSegments > 1 && (numSegments > 30 || random.nextInt(5) == 3)) {
+
       Collections.shuffle(segments, random);
 
       // TODO: sometimes make more than 1 merge?
       mergeSpec = new MergeSpecification();
-      final int segsToMerge = _TestUtil.nextInt(random, 1, segmentInfos.size());
+      final int segsToMerge = _TestUtil.nextInt(random, 1, numSegments);
       mergeSpec.add(new OneMerge(segments.subList(0, segsToMerge)));
     }
 
@@ -97,7 +111,7 @@ public class MockRandomMergePolicy exten
 
   @Override
   public MergeSpecification findForcedDeletesMerges(SegmentInfos segmentInfos) throws IOException {
-    return findMerges(segmentInfos);
+    return findMerges(null, segmentInfos);
   }
 
   @Override

Modified: lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java Thu Oct 25 13:10:25 2012
@@ -29,11 +29,10 @@ import java.util.Set;
 
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.codecs.asserting.AssertingPostingsFormat;
-import org.apache.lucene.codecs.block.BlockPostingsFormat;
-import org.apache.lucene.codecs.bloom.TestBloomFilteredLucene40Postings;
-import org.apache.lucene.codecs.lucene40.Lucene40Codec;
-import org.apache.lucene.codecs.lucene40.Lucene40PostingsFormat;
-import org.apache.lucene.codecs.lucene40ords.Lucene40WithOrds;
+import org.apache.lucene.codecs.lucene41.Lucene41Codec;
+import org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat;
+import org.apache.lucene.codecs.lucene41ords.Lucene41WithOrds;
+import org.apache.lucene.codecs.bloom.TestBloomFilteredLucene41Postings;
 import org.apache.lucene.codecs.memory.DirectPostingsFormat;
 import org.apache.lucene.codecs.memory.MemoryPostingsFormat;
 import org.apache.lucene.codecs.mockintblock.MockFixedIntBlockPostingsFormat;
@@ -41,7 +40,7 @@ import org.apache.lucene.codecs.mockintb
 import org.apache.lucene.codecs.mockrandom.MockRandomPostingsFormat;
 import org.apache.lucene.codecs.mocksep.MockSepPostingsFormat;
 import org.apache.lucene.codecs.nestedpulsing.NestedPulsingPostingsFormat;
-import org.apache.lucene.codecs.pulsing.Pulsing40PostingsFormat;
+import org.apache.lucene.codecs.pulsing.Pulsing41PostingsFormat;
 import org.apache.lucene.codecs.simpletext.SimpleTextPostingsFormat;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util._TestUtil;
@@ -55,7 +54,7 @@ import org.apache.lucene.util._TestUtil;
  * documents in different orders and the test will still be deterministic
  * and reproducable.
  */
-public class RandomCodec extends Lucene40Codec {
+public class RandomCodec extends Lucene41Codec {
   /** Shuffled list of postings formats to use for new mappings */
   private List<PostingsFormat> formats = new ArrayList<PostingsFormat>();
   
@@ -94,23 +93,22 @@ public class RandomCodec extends Lucene4
     int lowFreqCutoff = _TestUtil.nextInt(random, 2, 100);
 
     add(avoidCodecs,
-        new Lucene40PostingsFormat(minItemsPerBlock, maxItemsPerBlock),
-        new BlockPostingsFormat(minItemsPerBlock, maxItemsPerBlock),
+        new Lucene41PostingsFormat(minItemsPerBlock, maxItemsPerBlock),
         new DirectPostingsFormat(LuceneTestCase.rarely(random) ? 1 : (LuceneTestCase.rarely(random) ? Integer.MAX_VALUE : maxItemsPerBlock),
                                  LuceneTestCase.rarely(random) ? 1 : (LuceneTestCase.rarely(random) ? Integer.MAX_VALUE : lowFreqCutoff)),
-        new Pulsing40PostingsFormat(1 + random.nextInt(20), minItemsPerBlock, maxItemsPerBlock),
+        new Pulsing41PostingsFormat(1 + random.nextInt(20), minItemsPerBlock, maxItemsPerBlock),
         // add pulsing again with (usually) different parameters
-        new Pulsing40PostingsFormat(1 + random.nextInt(20), minItemsPerBlock, maxItemsPerBlock),
-        //TODO as a PostingsFormat which wraps others, we should allow TestBloomFilteredLucene40Postings to be constructed 
+        new Pulsing41PostingsFormat(1 + random.nextInt(20), minItemsPerBlock, maxItemsPerBlock),
+        //TODO as a PostingsFormat which wraps others, we should allow TestBloomFilteredLucene41Postings to be constructed 
         //with a choice of concrete PostingsFormats. Maybe useful to have a generic means of marking and dealing 
         //with such "wrapper" classes?
-        new TestBloomFilteredLucene40Postings(),                
+        new TestBloomFilteredLucene41Postings(),                
         new MockSepPostingsFormat(),
         new MockFixedIntBlockPostingsFormat(_TestUtil.nextInt(random, 1, 2000)),
         new MockVariableIntBlockPostingsFormat( _TestUtil.nextInt(random, 1, 127)),
         new MockRandomPostingsFormat(random),
         new NestedPulsingPostingsFormat(),
-        new Lucene40WithOrds(),
+        new Lucene41WithOrds(),
         new SimpleTextPostingsFormat(),
         new AssertingPostingsFormat(),
         new MemoryPostingsFormat(true, random.nextFloat()),

Modified: lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java Thu Oct 25 13:10:25 2012
@@ -126,8 +126,23 @@ public class RandomIndexWriter implement
     // any forced merges:
     doRandomForceMerge = r.nextBoolean();
   } 
+  
+  private boolean addDocValuesFields = true;
+  
+  /**
+   * set to false if you don't want RandomIndexWriter
+   * adding docvalues fields.
+   */
+  public void setAddDocValuesFields(boolean v) {
+    addDocValuesFields = v;
+    switchDoDocValues();
+  }
 
   private void switchDoDocValues() {
+    if (addDocValuesFields == false) {
+      doDocValues = false;
+      return;
+    }
     // randomly enable / disable docValues 
     doDocValues = LuceneTestCase.rarely(r);
     if (LuceneTestCase.VERBOSE) {
@@ -387,10 +402,16 @@ public class RandomIndexWriter implement
       final int segCount = w.getSegmentCount();
       if (r.nextBoolean() || segCount == 0) {
         // full forceMerge
+        if (LuceneTestCase.VERBOSE) {
+          System.out.println("RIW: doRandomForceMerge(1)");
+        }
         w.forceMerge(1);
       } else {
         // partial forceMerge
         final int limit = _TestUtil.nextInt(r, 1, segCount);
+        if (LuceneTestCase.VERBOSE) {
+          System.out.println("RIW: doRandomForceMerge(" + limit + ")");
+        }
         w.forceMerge(limit);
         assert !doRandomForceMergeAssert || w.getSegmentCount() <= limit: "limit=" + limit + " actual=" + w.getSegmentCount();
       }



Mime
View raw message