lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rm...@apache.org
Subject svn commit: r1629403 [1/4] - in /lucene/dev/branches/lucene5969: ./ dev-tools/ dev-tools/scripts/ lucene/ lucene/backward-codecs/ lucene/backward-codecs/src/test/org/apache/lucene/index/ lucene/core/ lucene/core/src/java/org/apache/lucene/document/ luc...
Date Sat, 04 Oct 2014 15:08:35 GMT
Author: rmuir
Date: Sat Oct  4 15:08:32 2014
New Revision: 1629403

URL: http://svn.apache.org/r1629403
Log:
merge trunk

Added:
    lucene/dev/branches/lucene5969/lucene/backward-codecs/src/test/org/apache/lucene/index/index.4.10.1-cfs.zip
      - copied unchanged from r1629401, lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/index/index.4.10.1-cfs.zip
    lucene/dev/branches/lucene5969/lucene/backward-codecs/src/test/org/apache/lucene/index/index.4.10.1-nocfs.zip
      - copied unchanged from r1629401, lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/index/index.4.10.1-nocfs.zip
    lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java
      - copied unchanged from r1629401, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java
    lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/SparseFixedBitSet.java
      - copied unchanged from r1629401, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/SparseFixedBitSet.java
    lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/util/TestDocIdSetBuilder.java
      - copied unchanged from r1629401, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/TestDocIdSetBuilder.java
    lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/util/TestSparseFixedBitSet.java
      - copied unchanged from r1629401, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/TestSparseFixedBitSet.java
    lucene/dev/branches/lucene5969/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
      - copied unchanged from r1629401, lucene/dev/trunk/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
    lucene/dev/branches/lucene5969/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java
      - copied unchanged from r1629401, lucene/dev/trunk/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java
    lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/rest/schema/SchemaZkVersionResource.java
      - copied unchanged from r1629401, lucene/dev/trunk/solr/core/src/java/org/apache/solr/rest/schema/SchemaZkVersionResource.java
    lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/schema/SchemaManager.java
      - copied unchanged from r1629401, lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SchemaManager.java
    lucene/dev/branches/lucene5969/solr/core/src/test-files/solr/collection1/conf/solrconfig-nocache-with-delaying-searchcomponent.xml
      - copied unchanged from r1629401, lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-nocache-with-delaying-searchcomponent.xml
    lucene/dev/branches/lucene5969/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog-with-delayingcomponent.xml
      - copied unchanged from r1629401, lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog-with-delayingcomponent.xml
    lucene/dev/branches/lucene5969/solr/core/src/test/org/apache/solr/cloud/LeaderInitiatedRecoveryOnCommitTest.java
      - copied unchanged from r1629401, lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/LeaderInitiatedRecoveryOnCommitTest.java
    lucene/dev/branches/lucene5969/solr/core/src/test/org/apache/solr/rest/schema/TestBulkSchemaAPI.java
      - copied unchanged from r1629401, lucene/dev/trunk/solr/core/src/test/org/apache/solr/rest/schema/TestBulkSchemaAPI.java
    lucene/dev/branches/lucene5969/solr/core/src/test/org/apache/solr/schema/TestBulkSchemaConcurrent.java
      - copied unchanged from r1629401, lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/TestBulkSchemaConcurrent.java
    lucene/dev/branches/lucene5969/solr/core/src/test/org/apache/solr/schema/TestSchemaManager.java
      - copied unchanged from r1629401, lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/TestSchemaManager.java
    lucene/dev/branches/lucene5969/solr/core/src/test/org/apache/solr/search/DelayingSearchComponent.java
      - copied unchanged from r1629401, lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/DelayingSearchComponent.java
Removed:
    lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/ConstantScoreAutoRewrite.java
    lucene/dev/branches/lucene5969/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java
    lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/ChainedFilter.java
    lucene/dev/branches/lucene5969/lucene/queries/src/test/org/apache/lucene/queries/ChainedFilterTest.java
Modified:
    lucene/dev/branches/lucene5969/   (props changed)
    lucene/dev/branches/lucene5969/dev-tools/   (props changed)
    lucene/dev/branches/lucene5969/dev-tools/scripts/checkJavaDocs.py
    lucene/dev/branches/lucene5969/lucene/   (props changed)
    lucene/dev/branches/lucene5969/lucene/CHANGES.txt   (contents, props changed)
    lucene/dev/branches/lucene5969/lucene/backward-codecs/   (props changed)
    lucene/dev/branches/lucene5969/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
    lucene/dev/branches/lucene5969/lucene/core/   (props changed)
    lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/document/FieldType.java
    lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
    lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
    lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/FieldInvertState.java
    lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java
    lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
    lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/MultiTermQuery.java
    lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
    lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/NumericRangeQuery.java
    lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/PrefixQuery.java
    lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/ScoringRewrite.java
    lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/TermRangeQuery.java
    lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java
    lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java
    lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/Version.java
    lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BBinaryDocValues.java
    lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BNumericDocValues.java
    lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BSortedDocValues.java
    lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BTerms.java
    lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java
    lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
    lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
    lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java
    lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestReaderClosed.java
    lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestAutomatonQuery.java
    lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestAutomatonQueryUnicode.java
    lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java
    lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestMultiTermQueryRewrites.java
    lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestWildcard.java
    lucene/dev/branches/lucene5969/lucene/demo/   (props changed)
    lucene/dev/branches/lucene5969/lucene/demo/src/java/org/apache/lucene/demo/facet/AssociationsFacetsExample.java
    lucene/dev/branches/lucene5969/lucene/demo/src/java/org/apache/lucene/demo/facet/DistanceFacetsExample.java
    lucene/dev/branches/lucene5969/lucene/demo/src/java/org/apache/lucene/demo/facet/ExpressionAggregationFacetsExample.java
    lucene/dev/branches/lucene5969/lucene/demo/src/java/org/apache/lucene/demo/facet/MultiCategoryListsFacetsExample.java
    lucene/dev/branches/lucene5969/lucene/demo/src/java/org/apache/lucene/demo/facet/RangeFacetsExample.java
    lucene/dev/branches/lucene5969/lucene/demo/src/java/org/apache/lucene/demo/facet/SimpleFacetsExample.java
    lucene/dev/branches/lucene5969/lucene/demo/src/java/org/apache/lucene/demo/facet/SimpleSortedSetFacetsExample.java
    lucene/dev/branches/lucene5969/lucene/memory/   (props changed)
    lucene/dev/branches/lucene5969/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
    lucene/dev/branches/lucene5969/lucene/queries/   (props changed)
    lucene/dev/branches/lucene5969/lucene/queries/src/java/org/apache/lucene/queries/TermsFilter.java
    lucene/dev/branches/lucene5969/lucene/queries/src/test/org/apache/lucene/queries/TermsFilterTest.java
    lucene/dev/branches/lucene5969/lucene/queryparser/   (props changed)
    lucene/dev/branches/lucene5969/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java
    lucene/dev/branches/lucene5969/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/CommonQueryParserConfiguration.java
    lucene/dev/branches/lucene5969/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/StandardQueryParser.java
    lucene/dev/branches/lucene5969/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/StandardQueryConfigHandler.java
    lucene/dev/branches/lucene5969/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/MultiTermRewriteMethodProcessor.java
    lucene/dev/branches/lucene5969/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java
    lucene/dev/branches/lucene5969/lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java
    lucene/dev/branches/lucene5969/lucene/replicator/   (props changed)
    lucene/dev/branches/lucene5969/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyReplicationClientTest.java
    lucene/dev/branches/lucene5969/lucene/sandbox/   (props changed)
    lucene/dev/branches/lucene5969/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/DuplicateFilter.java
    lucene/dev/branches/lucene5969/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedTermRangeQuery.java
    lucene/dev/branches/lucene5969/lucene/test-framework/   (props changed)
    lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/util/BaseDocIdSetTestCase.java
    lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java
    lucene/dev/branches/lucene5969/solr/   (props changed)
    lucene/dev/branches/lucene5969/solr/CHANGES.txt   (contents, props changed)
    lucene/dev/branches/lucene5969/solr/core/   (props changed)
    lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/cloud/Overseer.java
    lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
    lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/cloud/ZkController.java
    lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java
    lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/core/SolrConfig.java
    lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
    lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java
    lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/rest/BaseSolrResource.java
    lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/rest/SolrSchemaRestApi.java
    lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/rest/schema/BaseFieldResource.java
    lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/rest/schema/BaseFieldTypeResource.java
    lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/rest/schema/CopyFieldCollectionResource.java
    lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/rest/schema/DynamicFieldCollectionResource.java
    lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/rest/schema/DynamicFieldResource.java
    lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/rest/schema/FieldCollectionResource.java
    lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/rest/schema/FieldResource.java
    lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/rest/schema/FieldTypeCollectionResource.java
    lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/rest/schema/FieldTypeResource.java
    lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/rest/schema/SchemaResource.java
    lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/schema/CurrencyField.java
    lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/schema/FieldType.java
    lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
    lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java
    lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchemaFactory.java
    lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/schema/ZkIndexSchemaReader.java
    lucene/dev/branches/lucene5969/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
    lucene/dev/branches/lucene5969/solr/core/src/test/org/apache/solr/TestDistributedSearch.java
    lucene/dev/branches/lucene5969/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java
    lucene/dev/branches/lucene5969/solr/core/src/test/org/apache/solr/cloud/CloudExitableDirectoryReaderTest.java
    lucene/dev/branches/lucene5969/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java
    lucene/dev/branches/lucene5969/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java
    lucene/dev/branches/lucene5969/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java
    lucene/dev/branches/lucene5969/solr/core/src/test/org/apache/solr/core/ExitableDirectoryReaderTest.java
    lucene/dev/branches/lucene5969/solr/core/src/test/org/apache/solr/schema/TestCloudManagedSchemaConcurrent.java
    lucene/dev/branches/lucene5969/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java
    lucene/dev/branches/lucene5969/solr/solrj/   (props changed)
    lucene/dev/branches/lucene5969/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
    lucene/dev/branches/lucene5969/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
    lucene/dev/branches/lucene5969/solr/test-framework/   (props changed)
    lucene/dev/branches/lucene5969/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java

Modified: lucene/dev/branches/lucene5969/dev-tools/scripts/checkJavaDocs.py
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/dev-tools/scripts/checkJavaDocs.py?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/dev-tools/scripts/checkJavaDocs.py (original)
+++ lucene/dev/branches/lucene5969/dev-tools/scripts/checkJavaDocs.py Sat Oct  4 15:08:32 2014
@@ -161,9 +161,11 @@ def checkClassSummaries(fullPath):
 
   foundMethodDetail = False
   lastMethodAnchor = None
+  lineCount = 0
   
   for line in f.readlines():
     m = reMethodDetail.search(line)
+    lineCount += 1
     if m is not None:
       foundMethodDetail = True
       continue
@@ -212,6 +214,8 @@ def checkClassSummaries(fullPath):
     if inThing:
       if lineLower.find('</tr>') != -1:
         if not hasDesc:
+          if lastItem is None:
+            raise RuntimeError('failed to locate javadoc item in %s, line %d? last line: %s' % (fullPath, lineCount, line.rstrip()))
           missing.append((lastCaption, unEscapeURL(lastItem)))
         inThing = False
         continue

Modified: lucene/dev/branches/lucene5969/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/CHANGES.txt?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/lucene5969/lucene/CHANGES.txt Sat Oct  4 15:08:32 2014
@@ -70,6 +70,9 @@ New Features
   exiting requests that take too long to enumerate over terms. (Anshum Gupta, Steve Rowe,
   Robert Muir)
 
+* LUCENE-5911: Add MemoryIndex.freeze() to allow thread-safe searching over a 
+  MemoryIndex. (Alan Woodward)
+
 API Changes
 
 * LUCENE-5900: Deprecated more constructors taking Version in *InfixSuggester and
@@ -133,42 +136,30 @@ API Changes
 * LUCENE-5569: *AtomicReader/AtomicReaderContext have been renamed to *LeafReader/LeafReaderContext.
   (Ryan Ernst)
 
+* LUCENE-5938: Removed MultiTermQuery.ConstantScoreAutoRewrite as
+  MultiTermQuery.CONSTANT_SCORE_FILTER_REWRITE is usually better. (Adrien Grand)
+
+* LUCENE-5924: Rename CheckIndex -fix option to -exorcise. This option does not
+  actually fix the index, it just drops data.  (Robert Muir)
+
 Bug Fixes
 
 * LUCENE-5650: Enforce read-only access to any path outside the temporary
   folder via security manager, and make test temp dirs absolute.
   (Ryan Ernst, Dawid Weiss)
 
-* LUCENE-5904: Fixed a corruption case that can happen when 1)
-  IndexWriter is uncleanly shut-down (OS crash, power loss, etc.), 2)
-  on startup, when a new IndexWriter is created, a virus checker is
-  holding some of the previously written but unused files open and
-  preventing deletion, 3) IndexWriter writes these files again during
-  the course of indexing, then the files can later be deleted, causing
-  corruption.  This case was detected by adding evilness to
-  MockDirectoryWrapper to have it simulate a virus checker holding a
-  file open and preventing deletion (Robert Muir, Mike McCandless)
-
-* LUCENE-5916: Static scope test components should be consistent between
-  tests (and test iterations). Fix for FaultyIndexInput in particular.
-  (Dawid Weiss)
-
-* LUCENE-5934: Fix backwards compatibility for 4.0 indexes.
-  (Ian Lea, Uwe Schindler, Robert Muir, Ryan Ernst)
-
-* LUCENE-5958: Don't let exceptions during checkpoint corrupt the index.
-  Refactor existing OOM handling too, so you don't need to handle OOM special
-  for every IndexWriter method: instead such disasters will cause IW to close itself
-  defensively. (Robert Muir, Mike McCandless)
-
-* LUCENE-5952: Improve error messages when version cannot be parsed;
-  don't check for too old or too new major version (it's too low level
-  to enforce here); use simple string tokenizer.  (Ryan Ernst, Uwe Schindler,
-  Robert Muir, Mike McCandless)
-
 * LUCENE-5948: RateLimiter now fully inits itself on init.  (Varun
   Thacker via Mike McCandless)
 
+* LUCENE-5977: Fix tokenstream safety checks in IndexWriter to properly
+  work across multi-valued fields. Previously some cases across multi-valued
+  fields would happily create a corrupt index. (Dawid Weiss, Robert Muir)
+
+* LUCENE-5981: CheckIndex obtains write.lock, since with some parameters it
+  may modify the index, and to prevent false corruption reports, as it does
+  not have the regular "spinlock" of DirectoryReader.open. It now implements
+  Closeable and you must close it to release the lock.  (Mike McCandless, Robert Muir)
+
 Documentation
 
 * LUCENE-5392: Add/improve analysis package documentation to reflect
@@ -176,13 +167,6 @@ Documentation
 
 Tests
 
-* LUCENE-5936: Add backcompat checks to verify what is tested matches known versions
-  (Ryan Ernst)
-
-* LUCENE-5939: Regenerate old backcompat indexes to ensure they were built with
-  the exact release
-  (Ryan Ernst, Uwe Schindler)
-
 * LUCENE-5957: Add option for tests to not randomize codec
   (Ryan Ernst)
 
@@ -205,6 +189,10 @@ Optimizations
 * LUCENE-5963: Reduce memory allocations in
   AnalyzingSuggester. (Markus Heiden via Mike McCandless)
 
+* LUCENE-5938: MultiTermQuery.CONSTANT_SCORE_FILTER_REWRITE is now faster on
+  queries that match few documents by using a sparse bit set implementation.
+  (Adrien Grand)
+
 Build
 
 * LUCENE-5909: Smoke tester now has better command line parsing and
@@ -225,6 +213,50 @@ Other
 
 * LUCENE-5915: Remove Pulsing postings format. (Robert Muir)
 
+======================= Lucene 4.10.1 ======================
+
+Bug fixes
+
+* LUCENE-5934: Fix backwards compatibility for 4.0 indexes.
+  (Ian Lea, Uwe Schindler, Robert Muir, Ryan Ernst)
+
+* LUCENE-5939: Regenerate old backcompat indexes to ensure they were built with
+  the exact release
+  (Ryan Ernst, Uwe Schindler)
+
+* LUCENE-5952: Improve error messages when version cannot be parsed;
+  don't check for too old or too new major version (it's too low level
+  to enforce here); use simple string tokenizer.  (Ryan Ernst, Uwe Schindler,
+  Robert Muir, Mike McCandless)
+
+* LUCENE-5958: Don't let exceptions during checkpoint corrupt the index. 
+  Refactor existing OOM handling too, so you don't need to handle OOM special
+  for every IndexWriter method: instead such disasters will cause IW to close itself
+  defensively. (Robert Muir, Mike McCandless)
+
+* LUCENE-5904: Fixed a corruption case that can happen when 1)
+  IndexWriter is uncleanly shut-down (OS crash, power loss, etc.), 2)
+  on startup, when a new IndexWriter is created, a virus checker is
+  holding some of the previously written but unused files open and
+  preventing deletion, 3) IndexWriter writes these files again during
+  the course of indexing, then the files can later be deleted, causing
+  corruption.  This case was detected by adding evilness to
+  MockDirectoryWrapper to have it simulate a virus checker holding a
+  file open and preventing deletion (Robert Muir, Mike McCandless)
+
+* LUCENE-5916: Static scope test components should be consistent between
+  tests (and test iterations). Fix for FaultyIndexInput in particular.
+  (Dawid Weiss)
+
+* LUCENE-5975: Fix reading of 3.0-3.3 indexes, where bugs in these old
+  index formats would result in CorruptIndexException "did not read all
+  bytes from file" when reading the deleted docs file. (Patrick Mi, Robert MUir)
+
+Tests
+
+* LUCENE-5936: Add backcompat checks to verify what is tested matches known versions
+  (Ryan Ernst)
+
 ======================= Lucene 4.10.0 ======================
 
 New Features

Modified: lucene/dev/branches/lucene5969/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (original)
+++ lucene/dev/branches/lucene5969/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Sat Oct  4 15:08:32 2014
@@ -250,7 +250,9 @@ public class TestBackwardsCompatibility 
       "4.9.1-cfs",
       "4.9.1-nocfs",
       "4.10.0-cfs",
-      "4.10.0-nocfs"
+      "4.10.0-nocfs",
+      "4.10.1-cfs",
+      "4.10.1-nocfs"
   };
   
   final String[] unsupportedNames = {
@@ -514,6 +516,7 @@ public class TestBackwardsCompatibility 
       CheckIndex.Status indexStatus = checker.checkIndex();
       assertFalse(indexStatus.clean);
       assertTrue(bos.toString(IOUtils.UTF_8).contains(IndexFormatTooOldException.class.getName()));
+      checker.close();
 
       dir.close();
       IOUtils.rm(oldIndexDir);
@@ -749,12 +752,6 @@ public class TestBackwardsCompatibility 
     reader.close();
   }
 
-  private int compare(String name, String v) {
-    int v0 = Integer.parseInt(name.substring(0, 2));
-    int v1 = Integer.parseInt(v);
-    return v0 - v1;
-  }
-
   public void changeIndexWithAdds(Random random, Directory dir, Version nameVersion) throws IOException {
     // open writer
     IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random))
@@ -1023,7 +1020,7 @@ public class TestBackwardsCompatibility 
       assertTrue("codec used in " + name + " (" + codec.getName() + ") is not a default codec (does not begin with Lucene)",
                  codec.getName().startsWith("Lucene"));
     }
-      r.close();
+    r.close();
   }
   
   public void testAllIndexesUseDefaultCodec() throws Exception {
@@ -1186,7 +1183,7 @@ public class TestBackwardsCompatibility 
         // only use Log- or TieredMergePolicy, to make document addition predictable and not suddenly merge:
         MergePolicy mp = random().nextBoolean() ? newLogMergePolicy() : newTieredMergePolicy();
         IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()))
-          .setMergePolicy(mp).setCommitOnClose(false);
+          .setMergePolicy(mp);
         IndexWriter w = new IndexWriter(ramDir, iwc);
         // add few more docs:
         for(int j = 0; j < RANDOM_MULTIPLIER * random().nextInt(30); j++) {
@@ -1203,7 +1200,7 @@ public class TestBackwardsCompatibility 
       // version) to single segment index
       MergePolicy mp = random().nextBoolean() ? newLogMergePolicy() : newTieredMergePolicy();
       IndexWriterConfig iwc = new IndexWriterConfig(null)
-        .setMergePolicy(mp).setCommitOnClose(false);
+        .setMergePolicy(mp);
       IndexWriter w = new IndexWriter(dir, iwc);
       w.addIndexes(ramDir);
       try {

Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/document/FieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/document/FieldType.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/document/FieldType.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/document/FieldType.java Sat Oct  4 15:08:32 2014
@@ -424,7 +424,7 @@ public class FieldType implements Indexa
   }
 
   /**
-   * Set's the field's DocValuesType
+   * Sets the field's DocValuesType
    * @param type DocValues type, or null if no DocValues should be stored.
    * @throws IllegalStateException if this FieldType is frozen against
    *         future modifications.

Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java Sat Oct  4 15:08:32 2014
@@ -17,12 +17,14 @@ package org.apache.lucene.index;
  * limitations under the License.
  */
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.io.PrintStream;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.text.NumberFormat;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
@@ -35,10 +37,13 @@ import org.apache.lucene.codecs.blocktre
 import org.apache.lucene.index.CheckIndex.Status.DocValuesStatus;
 import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.Lock;
+import org.apache.lucene.store.LockObtainFailedException;
 import org.apache.lucene.util.Accountables;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
@@ -60,12 +65,14 @@ import org.apache.lucene.util.Version;
  * index it can take quite a long time to run.
  *
  * @lucene.experimental Please make a complete backup of your
- * index before using this to fix your index!
+ * index before using this to exorcise corrupted documents from your index!
  */
-public class CheckIndex {
+public class CheckIndex implements Closeable {
 
   private PrintStream infoStream;
   private Directory dir;
+  private Lock writeLock;
+  private volatile boolean closed;
 
   /**
    * Returned from {@link #checkIndex()} detailing the health and status of the index.
@@ -111,7 +118,7 @@ public class CheckIndex {
 
     /** 
      * SegmentInfos instance containing only segments that
-     * had no problems (this is used with the {@link CheckIndex#fixIndex} 
+     * had no problems (this is used with the {@link CheckIndex#exorciseIndex} 
      * method to repair the index. 
      */
     SegmentInfos newSegments;
@@ -342,9 +349,36 @@ public class CheckIndex {
   }
 
   /** Create a new CheckIndex on the directory. */
-  public CheckIndex(Directory dir) {
+  public CheckIndex(Directory dir) throws IOException {
+    this(dir, dir.makeLock(IndexWriter.WRITE_LOCK_NAME));
+  }
+  
+  /** 
+   * Expert: create a directory with the specified lock.
+   * This should really not be used except for unit tests!!!!
+   * It exists only to support special tests (such as TestIndexWriterExceptions*),
+   * that would otherwise be more complicated to debug if they had to close the writer
+   * for each check.
+   */
+  public CheckIndex(Directory dir, Lock writeLock) throws IOException {
     this.dir = dir;
-    infoStream = null;
+    this.writeLock = writeLock;
+    this.infoStream = null;
+    if (!writeLock.obtain(IndexWriterConfig.WRITE_LOCK_TIMEOUT)) { // obtain write lock
+      throw new LockObtainFailedException("Index locked for write: " + writeLock);
+    }
+  }
+  
+  private void ensureOpen() {
+    if (closed) {
+      throw new AlreadyClosedException("this instance is closed");
+    }
+  }
+
+  @Override
+  public void close() throws IOException {
+    closed = true;
+    IOUtils.close(writeLock);
   }
 
   private boolean crossCheckTermVectors;
@@ -415,18 +449,22 @@ public class CheckIndex {
    *
    *  <p>As this method checks every byte in the specified
    *  segments, on a large index it can take quite a long
-   *  time to run.
-   *
-   *  <p><b>WARNING</b>: make sure
-   *  you only call this when the index is not opened by any
-   *  writer. */
+   *  time to run. */
   public Status checkIndex(List<String> onlySegments) throws IOException {
+    ensureOpen();
     NumberFormat nf = NumberFormat.getInstance(Locale.ROOT);
     SegmentInfos sis = new SegmentInfos();
     Status result = new Status();
     result.dir = dir;
+    String[] files = dir.listAll();
+    String lastSegmentsFile = SegmentInfos.getLastCommitSegmentsFileName(files);
+    if (lastSegmentsFile == null) {
+      throw new IndexNotFoundException("no segments* file found in " + dir + ": files: " + Arrays.toString(files));
+    }
     try {
-      sis.read(dir);
+      // Do not use SegmentInfos.read(Directory) since the spooky
+      // retrying it does is not necessary here (we hold the write lock):
+      sis.read(dir, lastSegmentsFile);
     } catch (Throwable t) {
       if (failFast) {
         IOUtils.reThrow(t);
@@ -677,7 +715,7 @@ public class CheckIndex {
         }
         msg(infoStream, "FAILED");
         String comment;
-        comment = "fixIndex() would remove reference to this segment";
+        comment = "exorciseIndex() would remove reference to this segment";
         msg(infoStream, "    WARNING: " + comment + "; full exception:");
         if (infoStream != null)
           t.printStackTrace(infoStream);
@@ -2008,12 +2046,11 @@ public class CheckIndex {
    *  new segments file into the index, effectively removing
    *  all documents in broken segments from the index.
    *  BE CAREFUL.
-   *
-   * <p><b>WARNING</b>: Make sure you only call this when the
-   *  index is not opened  by any writer. */
-  public void fixIndex(Status result) throws IOException {
+   */
+  public void exorciseIndex(Status result) throws IOException {
+    ensureOpen();
     if (result.partial)
-      throw new IllegalArgumentException("can only fix an index that was fully checked (this status checked a subset of segments)");
+      throw new IllegalArgumentException("can only exorcise an index that was fully checked (this status checked a subset of segments)");
     result.newSegments.changed();
     result.newSegments.commit(result.dir);
   }
@@ -2030,31 +2067,31 @@ public class CheckIndex {
     return assertsOn;
   }
 
-  /** Command-line interface to check and fix an index.
+  /** Command-line interface to check and exorcise corrupt segments from an index.
 
     <p>
     Run it like this:
     <pre>
-    java -ea:org.apache.lucene... org.apache.lucene.index.CheckIndex pathToIndex [-fix] [-verbose] [-segment X] [-segment Y]
+    java -ea:org.apache.lucene... org.apache.lucene.index.CheckIndex pathToIndex [-exorcise] [-verbose] [-segment X] [-segment Y]
     </pre>
     <ul>
-    <li><code>-fix</code>: actually write a new segments_N file, removing any problematic segments
+    <li><code>-exorcise</code>: actually write a new segments_N file, removing any problematic segments. *LOSES DATA*
 
     <li><code>-segment X</code>: only check the specified
     segment(s).  This can be specified multiple times,
     to check more than one segment, eg <code>-segment _2
-    -segment _a</code>.  You can't use this with the -fix
+    -segment _a</code>.  You can't use this with the -exorcise
     option.
     </ul>
 
-    <p><b>WARNING</b>: <code>-fix</code> should only be used on an emergency basis as it will cause
+    <p><b>WARNING</b>: <code>-exorcise</code> should only be used on an emergency basis as it will cause
                        documents (perhaps many) to be permanently removed from the index.  Always make
                        a backup copy of your index before running this!  Do not run this tool on an index
                        that is actively being written to.  You have been warned!
 
-    <p>                Run without -fix, this tool will open the index, report version information
-                       and report any exceptions it hits and what action it would take if -fix were
-                       specified.  With -fix, this tool will remove any segments that have issues and
+    <p>                Run without -exorcise, this tool will open the index, report version information
+                       and report any exceptions it hits and what action it would take if -exorcise were
+                       specified.  With -exorcise, this tool will remove any segments that have issues and
                        write a new segments_N file.  This means all documents contained in the affected
                        segments will be removed.
 
@@ -2063,8 +2100,14 @@ public class CheckIndex {
                        corruption, else 0.
    */
   public static void main(String[] args) throws IOException, InterruptedException {
+    int exitCode = doMain(args);
+    System.exit(exitCode);
+  }
+  
+  // actual main: returns exit code instead of terminating JVM (for easy testing)
+  private static int doMain(String args[]) throws IOException, InterruptedException {
 
-    boolean doFix = false;
+    boolean doExorcise = false;
     boolean doCrossCheckTermVectors = false;
     boolean verbose = false;
     List<String> onlySegments = new ArrayList<>();
@@ -2073,8 +2116,8 @@ public class CheckIndex {
     int i = 0;
     while(i < args.length) {
       String arg = args[i];
-      if ("-fix".equals(arg)) {
-        doFix = true;
+      if ("-exorcise".equals(arg)) {
+        doExorcise = true;
       } else if ("-crossCheckTermVectors".equals(arg)) {
         doCrossCheckTermVectors = true;
       } else if (arg.equals("-verbose")) {
@@ -2082,21 +2125,21 @@ public class CheckIndex {
       } else if (arg.equals("-segment")) {
         if (i == args.length-1) {
           System.out.println("ERROR: missing name for -segment option");
-          System.exit(1);
+          return 1;
         }
         i++;
         onlySegments.add(args[i]);
       } else if ("-dir-impl".equals(arg)) {
         if (i == args.length - 1) {
           System.out.println("ERROR: missing value for -dir-impl option");
-          System.exit(1);
+          return 1;
         }
         i++;
         dirImpl = args[i];
       } else {
         if (indexPath != null) {
           System.out.println("ERROR: unexpected extra argument '" + args[i] + "'");
-          System.exit(1);
+          return 1;
         }
         indexPath = args[i];
       }
@@ -2105,32 +2148,32 @@ public class CheckIndex {
 
     if (indexPath == null) {
       System.out.println("\nERROR: index path not specified");
-      System.out.println("\nUsage: java org.apache.lucene.index.CheckIndex pathToIndex [-fix] [-crossCheckTermVectors] [-segment X] [-segment Y] [-dir-impl X]\n" +
+      System.out.println("\nUsage: java org.apache.lucene.index.CheckIndex pathToIndex [-exorcise] [-crossCheckTermVectors] [-segment X] [-segment Y] [-dir-impl X]\n" +
                          "\n" +
-                         "  -fix: actually write a new segments_N file, removing any problematic segments\n" +
+                         "  -exorcise: actually write a new segments_N file, removing any problematic segments\n" +
                          "  -crossCheckTermVectors: verifies that term vectors match postings; THIS IS VERY SLOW!\n" +
-                         "  -codec X: when fixing, codec to write the new segments_N file with\n" +
+                         "  -codec X: when exorcising, codec to write the new segments_N file with\n" +
                          "  -verbose: print additional details\n" +
                          "  -segment X: only check the specified segments.  This can be specified multiple\n" + 
                          "              times, to check more than one segment, eg '-segment _2 -segment _a'.\n" +
-                         "              You can't use this with the -fix option\n" +
+                         "              You can't use this with the -exorcise option\n" +
                          "  -dir-impl X: use a specific " + FSDirectory.class.getSimpleName() + " implementation. " +
                          "If no package is specified the " + FSDirectory.class.getPackage().getName() + " package will be used.\n" +
                          "\n" +
-                         "**WARNING**: -fix should only be used on an emergency basis as it will cause\n" +
+                         "**WARNING**: -exorcise *LOSES DATA*. This should only be used on an emergency basis as it will cause\n" +
                          "documents (perhaps many) to be permanently removed from the index.  Always make\n" +
                          "a backup copy of your index before running this!  Do not run this tool on an index\n" +
                          "that is actively being written to.  You have been warned!\n" +
                          "\n" +
-                         "Run without -fix, this tool will open the index, report version information\n" +
-                         "and report any exceptions it hits and what action it would take if -fix were\n" +
-                         "specified.  With -fix, this tool will remove any segments that have issues and\n" + 
+                         "Run without -exorcise, this tool will open the index, report version information\n" +
+                         "and report any exceptions it hits and what action it would take if -exorcise were\n" +
+                         "specified.  With -exorcise, this tool will remove any segments that have issues and\n" + 
                          "write a new segments_N file.  This means all documents contained in the affected\n" +
                          "segments will be removed.\n" +
                          "\n" +
                          "This tool exits with exit code 1 if the index cannot be opened or has any\n" +
                          "corruption, else 0.\n");
-      System.exit(1);
+      return 1;
     }
 
     if (!assertsOn())
@@ -2138,58 +2181,59 @@ public class CheckIndex {
 
     if (onlySegments.size() == 0)
       onlySegments = null;
-    else if (doFix) {
-      System.out.println("ERROR: cannot specify both -fix and -segment");
-      System.exit(1);
+    else if (doExorcise) {
+      System.out.println("ERROR: cannot specify both -exorcise and -segment");
+      return 1;
     }
 
     System.out.println("\nOpening index @ " + indexPath + "\n");
-    Directory dir = null;
+    Directory directory = null;
     Path path = Paths.get(indexPath);
     try {
       if (dirImpl == null) {
-        dir = FSDirectory.open(path);
+        directory = FSDirectory.open(path);
       } else {
-        dir = CommandLineUtil.newFSDirectory(dirImpl, path);
+        directory = CommandLineUtil.newFSDirectory(dirImpl, path);
       }
     } catch (Throwable t) {
       System.out.println("ERROR: could not open directory \"" + indexPath + "\"; exiting");
       t.printStackTrace(System.out);
-      System.exit(1);
-    }
-
-    CheckIndex checker = new CheckIndex(dir);
-    checker.setCrossCheckTermVectors(doCrossCheckTermVectors);
-    checker.setInfoStream(System.out, verbose);
-
-    Status result = checker.checkIndex(onlySegments);
-    if (result.missingSegments) {
-      System.exit(1);
+      return 1;
     }
 
-    if (!result.clean) {
-      if (!doFix) {
-        System.out.println("WARNING: would write new segments file, and " + result.totLoseDocCount + " documents would be lost, if -fix were specified\n");
+    try (Directory dir = directory;
+         CheckIndex checker = new CheckIndex(dir)) {
+      checker.setCrossCheckTermVectors(doCrossCheckTermVectors);
+      checker.setInfoStream(System.out, verbose);
+      
+      Status result = checker.checkIndex(onlySegments);
+      if (result.missingSegments) {
+        return 1;
+      }
+      
+      if (!result.clean) {
+        if (!doExorcise) {
+          System.out.println("WARNING: would write new segments file, and " + result.totLoseDocCount + " documents would be lost, if -exorcise were specified\n");
+        } else {
+          System.out.println("WARNING: " + result.totLoseDocCount + " documents will be lost\n");
+          System.out.println("NOTE: will write new segments file in 5 seconds; this will remove " + result.totLoseDocCount + " docs from the index. YOU WILL LOSE DATA. THIS IS YOUR LAST CHANCE TO CTRL+C!");
+          for(int s=0;s<5;s++) {
+            Thread.sleep(1000);
+            System.out.println("  " + (5-s) + "...");
+          }
+          System.out.println("Writing...");
+          checker.exorciseIndex(result);
+          System.out.println("OK");
+          System.out.println("Wrote new segments file \"" + result.newSegments.getSegmentsFileName() + "\"");
+        }
+      }
+      System.out.println("");
+      
+      if (result.clean == true) {
+        return 0;
       } else {
-        System.out.println("WARNING: " + result.totLoseDocCount + " documents will be lost\n");
-        System.out.println("NOTE: will write new segments file in 5 seconds; this will remove " + result.totLoseDocCount + " docs from the index. THIS IS YOUR LAST CHANCE TO CTRL+C!");
-        for(int s=0;s<5;s++) {
-          Thread.sleep(1000);
-          System.out.println("  " + (5-s) + "...");
-        }
-        System.out.println("Writing...");
-        checker.fixIndex(result);
-        System.out.println("OK");
-        System.out.println("Wrote new segments file \"" + result.newSegments.getSegmentsFileName() + "\"");
-      }
-    }
-    System.out.println("");
-
-    final int exitCode;
-    if (result.clean == true)
-      exitCode = 0;
-    else
-      exitCode = 1;
-    System.exit(exitCode);
+        return 1;
+      }
+    }
   }
 }

Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java Sat Oct  4 15:08:32 2014
@@ -550,7 +550,9 @@ final class DefaultIndexingChain extends
           fieldInfo.setNormValueType(FieldInfo.DocValuesType.NUMERIC);
           norms = new NormValuesWriter(fieldInfo, docState.docWriter.bytesUsed);
         }
-        norms.addValue(docState.docID, similarity.computeNorm(invertState));
+        if (invertState.length != 0) {
+          norms.addValue(docState.docID, similarity.computeNorm(invertState));
+        }
       }
 
       termsHashPerField.finish();
@@ -574,9 +576,6 @@ final class DefaultIndexingChain extends
       // TODO: after we fix analyzers, also check if termVectorOffsets will be indexed.
       final boolean checkOffsets = fieldType.indexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
 
-      int lastStartOffset = 0;
-      int lastPosition = 0;
-        
       /*
        * To assist people in tracking down problems in analysis components, we wish to write the field name to the infostream
        * when we fail. We expect some caller to eventually deal with the real exception, so we don't want any 'catch' clauses,
@@ -602,13 +601,13 @@ final class DefaultIndexingChain extends
 
           int posIncr = invertState.posIncrAttribute.getPositionIncrement();
           invertState.position += posIncr;
-          if (invertState.position < lastPosition) {
+          if (invertState.position < invertState.lastPosition) {
             if (posIncr == 0) {
               throw new IllegalArgumentException("first position increment must be > 0 (got 0) for field '" + field.name() + "'");
             }
             throw new IllegalArgumentException("position increments (and gaps) must be >= 0 (got " + posIncr + ") for field '" + field.name() + "'");
           }
-          lastPosition = invertState.position;
+          invertState.lastPosition = invertState.position;
           if (posIncr == 0) {
             invertState.numOverlap++;
           }
@@ -616,11 +615,11 @@ final class DefaultIndexingChain extends
           if (checkOffsets) {
             int startOffset = invertState.offset + invertState.offsetAttribute.startOffset();
             int endOffset = invertState.offset + invertState.offsetAttribute.endOffset();
-            if (startOffset < lastStartOffset || endOffset < startOffset) {
+            if (startOffset < invertState.lastStartOffset || endOffset < startOffset) {
               throw new IllegalArgumentException("startOffset must be non-negative, and endOffset must be >= startOffset, and offsets must not go backwards "
-                                                 + "startOffset=" + startOffset + ",endOffset=" + endOffset + ",lastStartOffset=" + lastStartOffset + " for field '" + field.name() + "'");
+                                                 + "startOffset=" + startOffset + ",endOffset=" + endOffset + ",lastStartOffset=" + invertState.lastStartOffset + " for field '" + field.name() + "'");
             }
-            lastStartOffset = startOffset;
+            invertState.lastStartOffset = startOffset;
           }
 
           //System.out.println("  term=" + invertState.termAttribute);

Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/FieldInvertState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/FieldInvertState.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/FieldInvertState.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/FieldInvertState.java Sat Oct  4 15:08:32 2014
@@ -39,6 +39,9 @@ public final class FieldInvertState {
   int maxTermFrequency;
   int uniqueTermCount;
   float boost;
+  // we must track these across field instances (multi-valued case)
+  int lastStartOffset = 0;
+  int lastPosition = 0;
   AttributeSource attributeSource;
 
   OffsetAttribute offsetAttribute;
@@ -74,6 +77,8 @@ public final class FieldInvertState {
     maxTermFrequency = 0;
     uniqueTermCount = 0;
     boost = 1.0f;
+    lastStartOffset = 0;
+    lastPosition = 0;
   }
   
   // TODO: better name?

Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java Sat Oct  4 15:08:32 2014
@@ -65,6 +65,9 @@ public class FilterLeafReader extends Le
      * @param in the underlying Fields instance.
      */
     public FilterFields(Fields in) {
+      if (in == null) {
+        throw new NullPointerException("incoming Fields cannot be null");
+      }
       this.in = in;
     }
 
@@ -98,6 +101,9 @@ public class FilterLeafReader extends Le
      * @param in the underlying Terms instance.
      */
     public FilterTerms(Terms in) {
+      if (in == null) {
+        throw new NullPointerException("incoming Terms cannot be null");
+      }
       this.in = in;
     }
 
@@ -156,7 +162,12 @@ public class FilterLeafReader extends Le
      * Creates a new FilterTermsEnum
      * @param in the underlying TermsEnum instance.
      */
-    public FilterTermsEnum(TermsEnum in) { this.in = in; }
+    public FilterTermsEnum(TermsEnum in) {
+      if (in == null) {
+        throw new NullPointerException("incoming TermsEnum cannot be null");
+      }
+      this.in = in;
+    }
 
     @Override
     public AttributeSource attributes() {
@@ -219,6 +230,9 @@ public class FilterLeafReader extends Le
      * @param in the underlying DocsEnum instance.
      */
     public FilterDocsEnum(DocsEnum in) {
+      if (in == null) {
+        throw new NullPointerException("incoming DocsEnum cannot be null");
+      }
       this.in = in;
     }
 
@@ -263,6 +277,9 @@ public class FilterLeafReader extends Le
      * @param in the underlying DocsAndPositionsEnum instance.
      */
     public FilterDocsAndPositionsEnum(DocsAndPositionsEnum in) {
+      if (in == null) {
+        throw new NullPointerException("incoming DocsAndPositionsEnum cannot be null");
+      }
       this.in = in;
     }
 
@@ -327,6 +344,9 @@ public class FilterLeafReader extends Le
    */
   public FilterLeafReader(LeafReader in) {
     super();
+    if (in == null) {
+      throw new NullPointerException("incoming LeafReader cannot be null");
+    }
     this.in = in;
     in.registerParentReader(this);
   }

Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Sat Oct  4 15:08:32 2014
@@ -794,7 +794,15 @@ public class IndexWriter implements Clos
         // segments) pending:
         changed();
       } else {
-        segmentInfos.read(directory);
+        String[] files = directory.listAll();
+        String lastSegmentsFile = SegmentInfos.getLastCommitSegmentsFileName(files);
+        if (lastSegmentsFile == null) {
+          throw new IndexNotFoundException("no segments* file found in " + directory + ": files: " + Arrays.toString(files));
+        }
+
+        // Do not use SegmentInfos.read(Directory) since the spooky
+        // retrying it does is not necessary here (we hold the write lock):
+        segmentInfos.read(directory, lastSegmentsFile);
 
         IndexCommit commit = config.getIndexCommit();
         if (commit != null) {

Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/MultiTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/MultiTermQuery.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/MultiTermQuery.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/MultiTermQuery.java Sat Oct  4 15:08:32 2014
@@ -49,8 +49,8 @@ import org.apache.lucene.util.AttributeS
  * prevents this.
  *
  * <p>The recommended rewrite method is {@link
- * #CONSTANT_SCORE_AUTO_REWRITE_DEFAULT}: it doesn't spend CPU
- * computing unhelpful scores, and it tries to pick the most
+ * #CONSTANT_SCORE_FILTER_REWRITE}: it doesn't spend CPU
+ * computing unhelpful scores, and is the most
  * performant rewrite method given the query. If you
  * need scoring (like {@link FuzzyQuery}, use
  * {@link TopTermsScoringBooleanQueryRewrite} which uses
@@ -58,12 +58,12 @@ import org.apache.lucene.util.AttributeS
  * and not hit this limitation.
  *
  * Note that org.apache.lucene.queryparser.classic.QueryParser produces
- * MultiTermQueries using {@link
- * #CONSTANT_SCORE_AUTO_REWRITE_DEFAULT} by default.
+ * MultiTermQueries using {@link #CONSTANT_SCORE_FILTER_REWRITE}
+ * by default.
  */
 public abstract class MultiTermQuery extends Query {
   protected final String field;
-  protected RewriteMethod rewriteMethod = CONSTANT_SCORE_AUTO_REWRITE_DEFAULT;
+  protected RewriteMethod rewriteMethod = CONSTANT_SCORE_FILTER_REWRITE;
 
   /** Abstract class that defines how the query is rewritten. */
   public static abstract class RewriteMethod {
@@ -104,7 +104,7 @@ public abstract class MultiTermQuery ext
    *  query.  Note that typically such scores are
    *  meaningless to the user, and require non-trivial CPU
    *  to compute, so it's almost always better to use {@link
-   *  #CONSTANT_SCORE_AUTO_REWRITE_DEFAULT} instead.
+   *  #CONSTANT_SCORE_FILTER_REWRITE} instead.
    *
    *  <p><b>NOTE</b>: This rewrite method will hit {@link
    *  BooleanQuery.TooManyClauses} if the number of terms
@@ -208,40 +208,6 @@ public abstract class MultiTermQuery ext
       topLevel.add(q, BooleanClause.Occur.SHOULD);
     }
   }
-    
-  /** A rewrite method that tries to pick the best
-   *  constant-score rewrite method based on term and
-   *  document counts from the query.  If both the number of
-   *  terms and documents is small enough, then {@link
-   *  #CONSTANT_SCORE_BOOLEAN_QUERY_REWRITE} is used.
-   *  Otherwise, {@link #CONSTANT_SCORE_FILTER_REWRITE} is
-   *  used.
-   */
-  public static class ConstantScoreAutoRewrite extends org.apache.lucene.search.ConstantScoreAutoRewrite {}
-
-  /** Read-only default instance of {@link
-   *  ConstantScoreAutoRewrite}, with {@link
-   *  ConstantScoreAutoRewrite#setTermCountCutoff} set to
-   *  {@link
-   *  ConstantScoreAutoRewrite#DEFAULT_TERM_COUNT_CUTOFF}
-   *  and {@link
-   *  ConstantScoreAutoRewrite#setDocCountPercent} set to
-   *  {@link
-   *  ConstantScoreAutoRewrite#DEFAULT_DOC_COUNT_PERCENT}.
-   *  Note that you cannot alter the configuration of this
-   *  instance; you'll need to create a private instance
-   *  instead. */
-  public final static RewriteMethod CONSTANT_SCORE_AUTO_REWRITE_DEFAULT = new ConstantScoreAutoRewrite() {
-    @Override
-    public void setTermCountCutoff(int count) {
-      throw new UnsupportedOperationException("Please create a private instance");
-    }
-
-    @Override
-    public void setDocCountPercent(double percent) {
-      throw new UnsupportedOperationException("Please create a private instance");
-    }
-  };
 
   /**
    * Constructs a query matching terms that cannot be represented with a single

Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java Sat Oct  4 15:08:32 2014
@@ -19,14 +19,14 @@ package org.apache.lucene.search;
 
 import java.io.IOException;
 
-import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.DocsEnum;
 import org.apache.lucene.index.Fields;
 import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.util.FixedBitSet;
 import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.DocIdSetBuilder;
 
 /**
  * A wrapper for {@link MultiTermQuery}, that exposes its
@@ -43,7 +43,7 @@ import org.apache.lucene.util.Bits;
  * this is why it is not abstract.
  */
 public class MultiTermQueryWrapperFilter<Q extends MultiTermQuery> extends Filter {
-    
+
   protected final Q query;
 
   /**
@@ -52,7 +52,7 @@ public class MultiTermQueryWrapperFilter
   protected MultiTermQueryWrapperFilter(Q query) {
       this.query = query;
   }
-  
+
   @Override
   public String toString() {
     // query.toString should be ok for the filter, too, if the query boost is 1.0f
@@ -77,7 +77,7 @@ public class MultiTermQueryWrapperFilter
 
   /** Returns the field name for this query */
   public final String getField() { return query.getField(); }
-  
+
   /**
    * Returns a DocIdSet with documents that should be permitted in search
    * results.
@@ -99,24 +99,13 @@ public class MultiTermQueryWrapperFilter
 
     final TermsEnum termsEnum = query.getTermsEnum(terms);
     assert termsEnum != null;
-    if (termsEnum.next() != null) {
-      // fill into a FixedBitSet
-      final FixedBitSet bitSet = new FixedBitSet(context.reader().maxDoc());
-      DocsEnum docsEnum = null;
-      do {
-        // System.out.println("  iter termCount=" + termCount + " term=" +
-        // enumerator.term().toBytesString());
-        docsEnum = termsEnum.docs(acceptDocs, docsEnum, DocsEnum.FLAG_NONE);
-        int docid;
-        while ((docid = docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
-          bitSet.set(docid);
-        }
-      } while (termsEnum.next() != null);
-      // System.out.println("  done termCount=" + termCount);
 
-      return bitSet;
-    } else {
-      return null;
+    DocIdSetBuilder builder = new DocIdSetBuilder(context.reader().maxDoc());
+    DocsEnum docs = null;
+    while (termsEnum.next() != null) {
+      docs = termsEnum.docs(acceptDocs, docs, DocsEnum.FLAG_NONE);
+      builder.or(docs);
     }
+    return builder.build();
   }
 }

Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/NumericRangeQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/NumericRangeQuery.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/NumericRangeQuery.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/NumericRangeQuery.java Sat Oct  4 15:08:32 2014
@@ -72,7 +72,7 @@ import org.apache.lucene.index.Term; // 
  * details.
  *
  * <p>This query defaults to {@linkplain
- * MultiTermQuery#CONSTANT_SCORE_AUTO_REWRITE_DEFAULT}.
+ * MultiTermQuery#CONSTANT_SCORE_FILTER_REWRITE}.
  * With precision steps of &le;4, this query can be run with
  * one of the BooleanQuery rewrite methods without changing
  * BooleanQuery's default max clause count.

Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/PrefixQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/PrefixQuery.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/PrefixQuery.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/PrefixQuery.java Sat Oct  4 15:08:32 2014
@@ -29,7 +29,7 @@ import org.apache.lucene.util.ToStringUt
  * is built by QueryParser for input like <code>app*</code>.
  *
  * <p>This query uses the {@link
- * MultiTermQuery#CONSTANT_SCORE_AUTO_REWRITE_DEFAULT}
+ * MultiTermQuery#CONSTANT_SCORE_FILTER_REWRITE}
  * rewrite method. */
 public class PrefixQuery extends MultiTermQuery {
   private Term prefix;

Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/ScoringRewrite.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/ScoringRewrite.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/ScoringRewrite.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/ScoringRewrite.java Sat Oct  4 15:08:32 2014
@@ -45,7 +45,7 @@ public abstract class ScoringRewrite<Q e
    *  query.  Note that typically such scores are
    *  meaningless to the user, and require non-trivial CPU
    *  to compute, so it's almost always better to use {@link
-   *  MultiTermQuery#CONSTANT_SCORE_AUTO_REWRITE_DEFAULT} instead.
+   *  MultiTermQuery#CONSTANT_SCORE_FILTER_REWRITE} instead.
    *
    *  <p><b>NOTE</b>: This rewrite method will hit {@link
    *  BooleanQuery.TooManyClauses} if the number of terms

Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/TermRangeQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/TermRangeQuery.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/TermRangeQuery.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/TermRangeQuery.java Sat Oct  4 15:08:32 2014
@@ -35,7 +35,7 @@ import org.apache.lucene.util.ToStringUt
  * for numerical ranges; use {@link NumericRangeQuery} instead.
  *
  * <p>This query uses the {@link
- * MultiTermQuery#CONSTANT_SCORE_AUTO_REWRITE_DEFAULT}
+ * MultiTermQuery#CONSTANT_SCORE_FILTER_REWRITE}
  * rewrite method.
  * @since 2.9
  */

Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java Sat Oct  4 15:08:32 2014
@@ -35,7 +35,7 @@ import org.apache.lucene.util.automaton.
  * a Wildcard term should not start with the wildcard <code>*</code>
  * 
  * <p>This query uses the {@link
- * MultiTermQuery#CONSTANT_SCORE_AUTO_REWRITE_DEFAULT}
+ * MultiTermQuery#CONSTANT_SCORE_FILTER_REWRITE}
  * rewrite method.
  *
  * @see AutomatonQuery

Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java Sat Oct  4 15:08:32 2014
@@ -112,7 +112,7 @@ public class DefaultSimilarity extends T
       numTerms = state.getLength() - state.getNumOverlap();
     else
       numTerms = state.getLength();
-   return state.getBoost() * ((float) (1.0 / Math.sqrt(numTerms)));
+    return state.getBoost() * ((float) (1.0 / Math.sqrt(numTerms)));
   }
 
   /** Implemented as <code>sqrt(freq)</code>. */

Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/Version.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/Version.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/Version.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/Version.java Sat Oct  4 15:08:32 2014
@@ -166,13 +166,20 @@ public final class Version {
   public static final Version LUCENE_4_9_0 = new Version(4, 9, 0);
   
   /**
-   * Match settings and bugs in Lucene's 4.10 release.
+   * Match settings and bugs in Lucene's 4.10.0 release.
    * @deprecated (5.0) Use latest
    */
   @Deprecated
   public static final Version LUCENE_4_10_0 = new Version(4, 10, 0);
 
   /**
+   * Match settings and bugs in Lucene's 4.10.1 release.
+   * @deprecated (5.0) Use latest
+   */
+  @Deprecated
+  public static final Version LUCENE_4_10_1 = new Version(4, 10, 1);
+
+  /**
    * Match settings and bugs in Lucene's 5.0 release.
    * @deprecated (5.0) Use latest
    */

Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BBinaryDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BBinaryDocValues.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BBinaryDocValues.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BBinaryDocValues.java Sat Oct  4 15:08:32 2014
@@ -37,7 +37,7 @@ import com.carrotsearch.randomizedtestin
 @Monster("takes ~ 45 minutes")
 public class Test2BBinaryDocValues extends LuceneTestCase {
   
-  // indexes Integer.MAX_VALUE docs with a fixed binary field
+  // indexes IndexWriter.MAX_DOCS docs with a fixed binary field
   public void testFixedBinary() throws Exception {
     BaseDirectoryWrapper dir = newFSDirectory(createTempDir("2BFixedBinary"));
     if (dir instanceof MockDirectoryWrapper) {
@@ -58,7 +58,7 @@ public class Test2BBinaryDocValues exten
     BinaryDocValuesField dvField = new BinaryDocValuesField("dv", data);
     doc.add(dvField);
     
-    for (int i = 0; i < Integer.MAX_VALUE; i++) {
+    for (int i = 0; i < IndexWriter.MAX_DOCS; i++) {
       bytes[0] = (byte)(i >> 24);
       bytes[1] = (byte)(i >> 16);
       bytes[2] = (byte)(i >> 8);
@@ -96,7 +96,7 @@ public class Test2BBinaryDocValues exten
     dir.close();
   }
   
-  // indexes Integer.MAX_VALUE docs with a variable binary field
+  // indexes IndexWriter.MAX_DOCS docs with a variable binary field
   public void testVariableBinary() throws Exception {
     BaseDirectoryWrapper dir = newFSDirectory(createTempDir("2BVariableBinary"));
     if (dir instanceof MockDirectoryWrapper) {
@@ -118,7 +118,7 @@ public class Test2BBinaryDocValues exten
     BinaryDocValuesField dvField = new BinaryDocValuesField("dv", data);
     doc.add(dvField);
     
-    for (int i = 0; i < Integer.MAX_VALUE; i++) {
+    for (int i = 0; i < IndexWriter.MAX_DOCS; i++) {
       encoder.reset(bytes);
       encoder.writeVInt(i % 65535); // 1, 2, or 3 bytes
       data.length = encoder.getPosition();

Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BNumericDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BNumericDocValues.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BNumericDocValues.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BNumericDocValues.java Sat Oct  4 15:08:32 2014
@@ -34,7 +34,7 @@ import com.carrotsearch.randomizedtestin
 @Monster("takes ~ 30 minutes")
 public class Test2BNumericDocValues extends LuceneTestCase {
   
-  // indexes Integer.MAX_VALUE docs with an increasing dv field
+  // indexes IndexWriter.MAX_DOCS docs with an increasing dv field
   public void testNumerics() throws Exception {
     BaseDirectoryWrapper dir = newFSDirectory(createTempDir("2BNumerics"));
     if (dir instanceof MockDirectoryWrapper) {
@@ -53,7 +53,7 @@ public class Test2BNumericDocValues exte
     NumericDocValuesField dvField = new NumericDocValuesField("dv", 0);
     doc.add(dvField);
     
-    for (int i = 0; i < Integer.MAX_VALUE; i++) {
+    for (int i = 0; i < IndexWriter.MAX_DOCS; i++) {
       dvField.setLongValue(i);
       w.addDocument(doc);
       if (i % 100000 == 0) {

Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BSortedDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BSortedDocValues.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BSortedDocValues.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BSortedDocValues.java Sat Oct  4 15:08:32 2014
@@ -56,7 +56,7 @@ public class Test2BSortedDocValues exten
     SortedDocValuesField dvField = new SortedDocValuesField("dv", data);
     doc.add(dvField);
     
-    for (int i = 0; i < Integer.MAX_VALUE; i++) {
+    for (int i = 0; i < IndexWriter.MAX_DOCS; i++) {
       bytes[0] = (byte)(i >> 8);
       bytes[1] = (byte) i;
       w.addDocument(doc);
@@ -111,7 +111,7 @@ public class Test2BSortedDocValues exten
     SortedDocValuesField dvField = new SortedDocValuesField("dv", data);
     doc.add(dvField);
     
-    for (int i = 0; i < Integer.MAX_VALUE; i++) {
+    for (int i = 0; i < IndexWriter.MAX_DOCS; i++) {
       bytes[0] = (byte)(i >> 24);
       bytes[1] = (byte)(i >> 16);
       bytes[2] = (byte)(i >> 8);

Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BTerms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BTerms.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BTerms.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BTerms.java Sat Oct  4 15:08:32 2014
@@ -42,23 +42,22 @@ import org.apache.lucene.util.AttributeI
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCase.Monster;
 import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
+import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.TestUtil;
 import org.apache.lucene.util.TimeUnits;
-
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
 
 // NOTE: SimpleText codec will consume very large amounts of
 // disk (but, should run successfully).  Best to run w/
 // -Dtests.codec=<current codec>, and w/ plenty of RAM, eg:
 //
-//   ant test -Dtests.monster=true -Dtests.heapsize=8g
-//
-//   java -server -Xmx8g -d64 -cp .:lib/junit-4.10.jar:./build/classes/test:./build/classes/test-framework:./build/classes/java -Dlucene.version=4.0-dev -Dtests.directory=MMapDirectory -DtempDir=build -ea org.junit.runner.JUnitCore org.apache.lucene.index.Test2BTerms
+//   ant test -Dtests.monster=true -Dtests.heapsize=8g -Dtests.codec=Lucene50 -Dtestcase=Test2BTerms
 //
 @SuppressCodecs({ "SimpleText", "Memory", "Direct" })
 @Monster("very slow, use 8g heap")
 @TimeoutSuite(millis = 6 * TimeUnits.HOUR)
+@SuppressSysoutChecks(bugUrl="We.print.lots.o.stuff.on.purpose")
 public class Test2BTerms extends LuceneTestCase {
 
   private final static int TOKEN_LEN = 5;

Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java Sat Oct  4 15:08:32 2014
@@ -26,6 +26,7 @@ import java.util.ArrayList;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.LockObtainFailedException;
 import org.apache.lucene.analysis.CannedTokenStream;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.analysis.Token;
@@ -95,6 +96,7 @@ public class TestCheckIndex extends Luce
     onlySegments.add("_0");
     
     assertTrue(checker.checkIndex(onlySegments).clean == true);
+    checker.close();
     dir.close();
   }
   
@@ -115,4 +117,22 @@ public class TestCheckIndex extends Luce
     iw.close();
     dir.close(); // checkindex
   }
+  
+  public void testObtainsLock() throws IOException {
+    Directory dir = newDirectory();
+    IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
+    iw.addDocument(new Document());
+    iw.commit();
+    
+    // keep IW open...
+    try {
+      new CheckIndex(dir);
+      fail("should not have obtained write lock");
+    } catch (LockObtainFailedException expected) {
+      // ok
+    }
+    
+    iw.close();
+    dir.close();
+  }
 }

Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java Sat Oct  4 15:08:32 2014
@@ -33,6 +33,7 @@ import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 
 import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.CannedTokenStream;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.analysis.MockTokenFilter;
 import org.apache.lucene.analysis.MockTokenizer;
@@ -2783,6 +2784,7 @@ public class TestIndexWriter extends Luc
     checker.setInfoStream(new PrintStream(bos, false, IOUtils.UTF_8), false);
     CheckIndex.Status indexStatus = checker.checkIndex(null);
     String s = bos.toString(IOUtils.UTF_8);
+    checker.close();
     // Make sure CheckIndex didn't fail
     assertTrue(s, indexStatus != null && indexStatus.clean);
 
@@ -2799,5 +2801,19 @@ public class TestIndexWriter extends Luc
       ids.add(id);
     }
   }
+  
+  public void testEmptyNorm() throws Exception {
+    Directory d = newDirectory();
+    IndexWriter w = new IndexWriter(d, newIndexWriterConfig(new MockAnalyzer(random())));
+    Document doc = new Document();
+    doc.add(new TextField("foo", new CannedTokenStream()));
+    w.addDocument(doc);
+    w.commit();
+    w.close();
+    DirectoryReader r = DirectoryReader.open(d);
+    assertEquals(0, getOnlySegmentReader(r).getNormValues("foo").get(0));
+    r.close();
+    d.close();
+  }
 }
 

Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java Sat Oct  4 15:08:32 2014
@@ -1194,6 +1194,7 @@ public class TestIndexWriterDelete exten
     checker.setInfoStream(new PrintStream(bos, false, IOUtils.UTF_8), false);
     CheckIndex.Status indexStatus = checker.checkIndex(null);
     assertTrue(indexStatus.clean);
+    checker.close();
     String s = bos.toString(IOUtils.UTF_8);
 
     // Segment should have deletions:
@@ -1204,9 +1205,11 @@ public class TestIndexWriterDelete exten
     w.close();
 
     bos = new ByteArrayOutputStream(1024);
+    checker = new CheckIndex(dir);
     checker.setInfoStream(new PrintStream(bos, false, IOUtils.UTF_8), false);
     indexStatus = checker.checkIndex(null);
     assertTrue(indexStatus.clean);
+    checker.close();
     s = bos.toString(IOUtils.UTF_8);
     assertFalse(s.contains("has deletions"));
     dir.close();

Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java Sat Oct  4 15:08:32 2014
@@ -426,6 +426,16 @@ public class TestPostingsOffsets extends
       //expected
     }
   }
+  
+  public void testIllegalOffsetsAcrossFieldInstances() throws Exception {
+    try {
+      checkTokens(new Token[] { makeToken("use", 1, 150, 160) }, 
+                  new Token[] { makeToken("use", 1, 50, 60) });
+      fail();
+    } catch (IllegalArgumentException expected) {
+      //expected
+    }
+  }
    
   public void testBackwardsOffsets() throws Exception {
     try {
@@ -508,6 +518,33 @@ public class TestPostingsOffsets extends
   }
   // TODO: more tests with other possibilities
   
+  private void checkTokens(Token[] field1, Token[] field2) throws IOException {
+    Directory dir = newDirectory();
+    RandomIndexWriter riw = new RandomIndexWriter(random(), dir, iwc);
+    boolean success = false;
+    try {
+      FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
+      ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
+      // store some term vectors for the checkindex cross-check
+      ft.setStoreTermVectors(true);
+      ft.setStoreTermVectorPositions(true);
+      ft.setStoreTermVectorOffsets(true);
+     
+      Document doc = new Document();
+      doc.add(new Field("body", new CannedTokenStream(field1), ft));
+      doc.add(new Field("body", new CannedTokenStream(field2), ft));
+      riw.addDocument(doc);
+      riw.close();
+      success = true;
+    } finally {
+      if (success) {
+        IOUtils.close(dir);
+      } else {
+        IOUtils.closeWhileHandlingException(riw, dir);
+      }
+    }
+  }
+  
   private void checkTokens(Token[] tokens) throws IOException {
     Directory dir = newDirectory();
     RandomIndexWriter riw = new RandomIndexWriter(random(), dir, iwc);

Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestReaderClosed.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestReaderClosed.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestReaderClosed.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestReaderClosed.java Sat Oct  4 15:08:32 2014
@@ -17,6 +17,9 @@ package org.apache.lucene.index;
  * limitations under the License.
  */
 
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.RejectedExecutionException;
+
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.analysis.MockTokenizer;
 import org.apache.lucene.document.Document;
@@ -65,6 +68,9 @@ public class TestReaderClosed extends Lu
       searcher.search(query, 5);
     } catch (AlreadyClosedException ace) {
       // expected
+    } catch (RejectedExecutionException ree) {
+      // expected if the searcher has been created with threads since LuceneTestCase
+      // closes the thread-pool in a reader close listener
     }
   }
 
@@ -81,7 +87,14 @@ public class TestReaderClosed extends Lu
     reader.close(); // close original child reader
     try {
       searcher.search(query, 5);
-    } catch (AlreadyClosedException ace) {
+    } catch (Exception e) {
+      AlreadyClosedException ace = null;
+      for (Throwable t = e; t != null; t = t.getCause()) {
+        if (t instanceof AlreadyClosedException) {
+          ace = (AlreadyClosedException) t;
+        }
+      }
+      assertNotNull("Query failed, but not due to an AlreadyClosedException", ace);
       assertEquals(
         "this IndexReader cannot be used anymore as one of its child readers was closed",
         ace.getMessage()

Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestAutomatonQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestAutomatonQuery.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestAutomatonQuery.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestAutomatonQuery.java Sat Oct  4 15:08:32 2014
@@ -98,9 +98,6 @@ public class TestAutomatonQuery extends 
     
     query.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_BOOLEAN_QUERY_REWRITE);
     assertEquals(expected, automatonQueryNrHits(query));
-    
-    query.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_AUTO_REWRITE_DEFAULT);
-    assertEquals(expected, automatonQueryNrHits(query));
   }
   
   /**

Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestAutomatonQueryUnicode.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestAutomatonQueryUnicode.java?rev=1629403&r1=1629402&r2=1629403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestAutomatonQueryUnicode.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestAutomatonQueryUnicode.java Sat Oct  4 15:08:32 2014
@@ -110,9 +110,6 @@ public class TestAutomatonQueryUnicode e
 
     query.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_BOOLEAN_QUERY_REWRITE);
     assertEquals(expected, automatonQueryNrHits(query));
-
-    query.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_AUTO_REWRITE_DEFAULT);
-    assertEquals(expected, automatonQueryNrHits(query));
   }
 
   /**



Mime
View raw message