Return-Path: X-Original-To: apmail-lucene-commits-archive@www.apache.org Delivered-To: apmail-lucene-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 068E8BE55 for ; Mon, 2 Jan 2012 16:26:53 +0000 (UTC) Received: (qmail 7386 invoked by uid 500); 2 Jan 2012 16:26:52 -0000 Mailing-List: contact commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@lucene.apache.org Delivered-To: mailing list commits@lucene.apache.org Received: (qmail 7379 invoked by uid 99); 2 Jan 2012 16:26:52 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 02 Jan 2012 16:26:52 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 02 Jan 2012 16:26:42 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 68A5A23889FD; Mon, 2 Jan 2012 16:26:19 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Subject: svn commit: r1226460 [1/2] - in /lucene/dev/branches/solrcloud: ./ dev-tools/idea/lucene/contrib/ lucene/ lucene/contrib/ lucene/contrib/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/ lucene/src/java/org/apache/lucene/codecs/ lucene/src/java... Date: Mon, 02 Jan 2012 16:26:14 -0000 To: commits@lucene.apache.org From: markrmiller@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120102162619.68A5A23889FD@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: markrmiller Date: Mon Jan 2 16:26:10 2012 New Revision: 1226460 URL: http://svn.apache.org/viewvc?rev=1226460&view=rev Log: merge up to r1226236 Added: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestRandomStoredFields.java - copied unchanged from r1226348, lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestRandomStoredFields.java lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKBigramFilter.java - copied unchanged from r1226348, lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKBigramFilter.java lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKWidthFilter.java - copied unchanged from r1226348, lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKWidthFilter.java lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKAnalyzer.java - copied unchanged from r1226348, lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKAnalyzer.java lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKWidthFilter.java - copied unchanged from r1226348, lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKWidthFilter.java lucene/dev/branches/solrcloud/modules/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestWithCJKBigramFilter.java - copied unchanged from r1226348, lucene/dev/trunk/modules/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestWithCJKBigramFilter.java lucene/dev/branches/solrcloud/modules/facet/src/java/org/apache/lucene/facet/doc-files/ - copied from r1226348, lucene/dev/trunk/modules/facet/src/java/org/apache/lucene/facet/doc-files/ lucene/dev/branches/solrcloud/modules/facet/src/java/org/apache/lucene/facet/doc-files/prettify.css - copied unchanged from r1226348, lucene/dev/trunk/modules/facet/src/java/org/apache/lucene/facet/doc-files/prettify.css lucene/dev/branches/solrcloud/modules/facet/src/java/org/apache/lucene/facet/doc-files/prettify.js - copied unchanged from r1226348, lucene/dev/trunk/modules/facet/src/java/org/apache/lucene/facet/doc-files/prettify.js lucene/dev/branches/solrcloud/modules/facet/src/java/org/apache/lucene/facet/doc-files/userguide.html - copied unchanged from r1226348, lucene/dev/trunk/modules/facet/src/java/org/apache/lucene/facet/doc-files/userguide.html lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/analysis/CJKBigramFilterFactory.java - copied unchanged from r1226348, lucene/dev/trunk/solr/core/src/java/org/apache/solr/analysis/CJKBigramFilterFactory.java lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/analysis/CJKWidthFilterFactory.java - copied unchanged from r1226348, lucene/dev/trunk/solr/core/src/java/org/apache/solr/analysis/CJKWidthFilterFactory.java lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestCJKBigramFilterFactory.java - copied unchanged from r1226348, lucene/dev/trunk/solr/core/src/test/org/apache/solr/analysis/TestCJKBigramFilterFactory.java lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestCJKWidthFilterFactory.java - copied unchanged from r1226348, lucene/dev/trunk/solr/core/src/test/org/apache/solr/analysis/TestCJKWidthFilterFactory.java lucene/dev/branches/solrcloud/solr/test-framework/src/java/org/apache/solr/core/MockFSDirectoryFactory.java - copied unchanged from r1226348, lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/core/MockFSDirectoryFactory.java Removed: lucene/dev/branches/solrcloud/modules/facet/docs/ Modified: lucene/dev/branches/solrcloud/ (props changed) lucene/dev/branches/solrcloud/dev-tools/idea/lucene/contrib/ (props changed) lucene/dev/branches/solrcloud/lucene/ (props changed) lucene/dev/branches/solrcloud/lucene/CHANGES.txt lucene/dev/branches/solrcloud/lucene/build.xml lucene/dev/branches/solrcloud/lucene/common-build.xml lucene/dev/branches/solrcloud/lucene/contrib/CHANGES.txt lucene/dev/branches/solrcloud/lucene/contrib/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestSpanRegexQuery.java (props changed) lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/DocValuesWriterBase.java lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReaderIndex.java (props changed) lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene40/values/Bytes.java lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene40/values/FixedSortedBytesImpl.java lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene40/values/VarSortedBytesImpl.java lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene40/values/Writer.java lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/analysis/MockTokenizer.java lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/search/CheckHits.java lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/search/QueryUtils.java lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCase.java lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/codecs/lucene3x/TestSurrogates.java lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/codecs/lucene3x/TestTermInfosReaderIndex.java lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/codecs/lucene40/TestDocValues.java lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/codecs/pulsing/Test10KPulsings.java lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestCompoundFile.java lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDuelingCodecs.java lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestStressNRT.java lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestFieldCache.java lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestShardSearching.java lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestSort.java lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestTermVectors.java lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/spans/TestBasics.java lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/store/TestLockFactory.java lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKTokenizer.java lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKTokenizer.java lucene/dev/branches/solrcloud/modules/benchmark/ (props changed) lucene/dev/branches/solrcloud/modules/facet/ (props changed) lucene/dev/branches/solrcloud/modules/facet/build.xml lucene/dev/branches/solrcloud/modules/facet/src/java/org/apache/lucene/facet/package.html lucene/dev/branches/solrcloud/modules/facet/src/java/overview.html lucene/dev/branches/solrcloud/modules/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandlerRandom.java lucene/dev/branches/solrcloud/modules/queries/src/test/org/apache/lucene/queries/TestCustomScoreQuery.java lucene/dev/branches/solrcloud/modules/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeFilterBuilder.java (props changed) lucene/dev/branches/solrcloud/solr/ (props changed) lucene/dev/branches/solrcloud/solr/CHANGES.txt (contents, props changed) lucene/dev/branches/solrcloud/solr/LICENSE.txt (props changed) lucene/dev/branches/solrcloud/solr/NOTICE.txt (props changed) lucene/dev/branches/solrcloud/solr/README.txt (props changed) lucene/dev/branches/solrcloud/solr/build.xml (props changed) lucene/dev/branches/solrcloud/solr/client/ (props changed) lucene/dev/branches/solrcloud/solr/cloud-dev/ (props changed) lucene/dev/branches/solrcloud/solr/common-build.xml (contents, props changed) lucene/dev/branches/solrcloud/solr/contrib/ (props changed) lucene/dev/branches/solrcloud/solr/contrib/analysis-extras/src/test-files/analysis-extras/solr/conf/solrconfig-icucollate.xml lucene/dev/branches/solrcloud/solr/contrib/clustering/src/test-files/ (props changed) lucene/dev/branches/solrcloud/solr/contrib/clustering/src/test-files/clustering/solr/conf/solrconfig.xml lucene/dev/branches/solrcloud/solr/contrib/dataimporthandler-extras/src/java/ (props changed) lucene/dev/branches/solrcloud/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/conf/dataimport-solrconfig.xml lucene/dev/branches/solrcloud/solr/contrib/dataimporthandler/src/java/ (props changed) lucene/dev/branches/solrcloud/solr/contrib/dataimporthandler/src/test-files/ (props changed) lucene/dev/branches/solrcloud/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/contentstream-solrconfig.xml lucene/dev/branches/solrcloud/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-nodatasource-solrconfig.xml lucene/dev/branches/solrcloud/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-solrconfig.xml lucene/dev/branches/solrcloud/solr/contrib/dataimporthandler/src/test/org/ (props changed) lucene/dev/branches/solrcloud/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilderThreaded.java lucene/dev/branches/solrcloud/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestErrorHandling.java lucene/dev/branches/solrcloud/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java lucene/dev/branches/solrcloud/solr/contrib/extraction/src/test-files/extraction/solr/conf/solrconfig.xml lucene/dev/branches/solrcloud/solr/contrib/uima/src/java/ (props changed) lucene/dev/branches/solrcloud/solr/contrib/uima/src/test-files/ (props changed) lucene/dev/branches/solrcloud/solr/contrib/uima/src/test-files/uima/solr/conf/solrconfig.xml lucene/dev/branches/solrcloud/solr/core/ (props changed) lucene/dev/branches/solrcloud/solr/core/src/java/ (props changed) lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/analysis/CJKTokenizerFactory.java lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/RecoveryStrat.java lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/ZkController.java lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/handler/RequestHandlerUtils.java lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java lucene/dev/branches/solrcloud/solr/core/src/test/ (props changed) lucene/dev/branches/solrcloud/solr/core/src/test-files/solr/conf/bad_solrconfig.xml lucene/dev/branches/solrcloud/solr/core/src/test-files/solr/conf/solrconfig-SOLR-749.xml lucene/dev/branches/solrcloud/solr/core/src/test-files/solr/conf/solrconfig-basic.xml lucene/dev/branches/solrcloud/solr/core/src/test-files/solr/conf/solrconfig-delpolicy2.xml lucene/dev/branches/solrcloud/solr/core/src/test-files/solr/conf/solrconfig-lazywriter.xml lucene/dev/branches/solrcloud/solr/core/src/test-files/solr/conf/solrconfig-master.xml lucene/dev/branches/solrcloud/solr/core/src/test-files/solr/conf/solrconfig-master1.xml lucene/dev/branches/solrcloud/solr/core/src/test-files/solr/conf/solrconfig-master2.xml lucene/dev/branches/solrcloud/solr/core/src/test-files/solr/conf/solrconfig-master3.xml lucene/dev/branches/solrcloud/solr/core/src/test-files/solr/conf/solrconfig-mergepolicy.xml lucene/dev/branches/solrcloud/solr/core/src/test-files/solr/conf/solrconfig-repeater.xml lucene/dev/branches/solrcloud/solr/core/src/test-files/solr/conf/solrconfig-slave.xml lucene/dev/branches/solrcloud/solr/core/src/test-files/solr/conf/solrconfig-slave1.xml lucene/dev/branches/solrcloud/solr/core/src/test-files/solr/conf/solrconfig-xinclude.xml lucene/dev/branches/solrcloud/solr/core/src/test-files/solr/conf/solrconfig_codec.xml lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/TestGroupingSearch.java lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/analysis/TestCJKTokenizerFactory.java lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/AbstractZkTestCase.java lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/core/TestArbitraryIndexDir.java lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/spelling/suggest/SuggesterTest.java lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/update/TestIndexingPerformance.java lucene/dev/branches/solrcloud/solr/dev-tools/ (props changed) lucene/dev/branches/solrcloud/solr/example/ (props changed) lucene/dev/branches/solrcloud/solr/lib/ (props changed) lucene/dev/branches/solrcloud/solr/scripts/ (props changed) lucene/dev/branches/solrcloud/solr/site/ (props changed) lucene/dev/branches/solrcloud/solr/site-src/ (props changed) lucene/dev/branches/solrcloud/solr/solrj/ (props changed) lucene/dev/branches/solrcloud/solr/solrj/src/java/ (props changed) lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/params/UpdateParams.java lucene/dev/branches/solrcloud/solr/solrj/src/test-files/solrj/solr/conf/solrconfig-slave1.xml lucene/dev/branches/solrcloud/solr/solrj/src/test-files/solrj/solr/shared/conf/solrconfig.xml lucene/dev/branches/solrcloud/solr/solrj/src/test/org/apache/solr/client/ (props changed) lucene/dev/branches/solrcloud/solr/solrj/src/test/org/apache/solr/client/solrj/ (props changed) lucene/dev/branches/solrcloud/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java lucene/dev/branches/solrcloud/solr/solrj/src/test/org/apache/solr/client/solrj/TestLBHttpSolrServer.java lucene/dev/branches/solrcloud/solr/solrj/src/test/org/apache/solr/common/ (props changed) lucene/dev/branches/solrcloud/solr/solrj/src/test/org/apache/solr/common/util/ContentStreamTest.java lucene/dev/branches/solrcloud/solr/test-framework/ (props changed) lucene/dev/branches/solrcloud/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java lucene/dev/branches/solrcloud/solr/test-framework/src/java/org/apache/solr/core/MockDirectoryFactory.java lucene/dev/branches/solrcloud/solr/testlogging.properties (props changed) lucene/dev/branches/solrcloud/solr/webapp/ (props changed) Modified: lucene/dev/branches/solrcloud/lucene/CHANGES.txt URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/CHANGES.txt?rev=1226460&r1=1226459&r2=1226460&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/CHANGES.txt (original) +++ lucene/dev/branches/solrcloud/lucene/CHANGES.txt Mon Jan 2 16:26:10 2012 @@ -649,6 +649,10 @@ Optimizations * LUCENE-3643: FilteredQuery and IndexSearcher.search(Query, Filter,...) now optimize the special case query instanceof MatchAllDocsQuery to execute as ConstantScoreQuery. (Uwe Schindler) + +* LUCENE-3509: Added fasterButMoreRam option for docvalues. This option controls whether the space for packed ints + should be rounded up for better performance. This option only applies for docvalues types bytes fixed sorted + and bytes var sorted. (Simon Willnauer, Martijn van Groningen) Bug fixes Modified: lucene/dev/branches/solrcloud/lucene/build.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/build.xml?rev=1226460&r1=1226459&r2=1226460&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/build.xml (original) +++ lucene/dev/branches/solrcloud/lucene/build.xml Mon Jan 2 16:26:10 2012 @@ -150,14 +150,10 @@ - - - - - - - - + + + + Modified: lucene/dev/branches/solrcloud/lucene/common-build.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/common-build.xml?rev=1226460&r1=1226459&r2=1226460&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/common-build.xml (original) +++ lucene/dev/branches/solrcloud/lucene/common-build.xml Mon Jan 2 16:26:10 2012 @@ -654,14 +654,10 @@ - - - - - - - - + + + + Modified: lucene/dev/branches/solrcloud/lucene/contrib/CHANGES.txt URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/contrib/CHANGES.txt?rev=1226460&r1=1226459&r2=1226460&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/contrib/CHANGES.txt (original) +++ lucene/dev/branches/solrcloud/lucene/contrib/CHANGES.txt Mon Jan 2 16:26:10 2012 @@ -102,6 +102,12 @@ New Features * SOLR-2982: Added phonetic encoders to contrib/analyzers/phonetic: Metaphone, Soundex, Caverphone, Beider-Morse, etc. (Robert Muir) +* LUCENE-2906: Added CJKBigramFilter that forms bigrams from StandardTokenizer or + ICUTokenizer CJK tokens, and CJKWidthFilter that normalizes halfwidth/fullwidth. + This filter supports unicode supplementary characters and you can toggle whether + bigrams are formed for each of Han/Hiragana/Katakana/Hangul independently. Deprecates + CJKTokenizer. (Tom Burton-West, Robert Muir) + API Changes * LUCENE-3596: DirectoryTaxonomyWriter.openIndexWriter() now takes an Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/DocValuesWriterBase.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/DocValuesWriterBase.java?rev=1226460&r1=1226459&r2=1226460&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/DocValuesWriterBase.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/DocValuesWriterBase.java Mon Jan 2 16:26:10 2012 @@ -24,6 +24,7 @@ import org.apache.lucene.codecs.lucene40 import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.PerDocWriteState; import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocValues.Type; // javadoc import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.util.BytesRef; @@ -31,6 +32,7 @@ import org.apache.lucene.util.Counter; /** * Abstract base class for PerDocConsumer implementations + * * @lucene.experimental */ //TODO: this needs to go under lucene40 codec (its specific to its impl) @@ -39,12 +41,27 @@ public abstract class DocValuesWriterBas protected final String segmentSuffix; private final Counter bytesUsed; protected final IOContext context; - + private final boolean fasterButMoreRam; + + /** + * @param state The state to initiate a {@link PerDocConsumer} instance + */ protected DocValuesWriterBase(PerDocWriteState state) { + this(state, true); + } + + /** + * @param state The state to initiate a {@link PerDocConsumer} instance + * @param fasterButMoreRam whether packed ints for docvalues should be optimized for speed by rounding up the bytes + * used for a value to either 8, 16, 32 or 64 bytes. This option is only applicable for + * docvalues of type {@link Type#BYTES_FIXED_SORTED} and {@link Type#BYTES_VAR_SORTED}. + */ + protected DocValuesWriterBase(PerDocWriteState state, boolean fasterButMoreRam) { this.segmentName = state.segmentName; this.segmentSuffix = state.segmentSuffix; this.bytesUsed = state.bytesUsed; this.context = state.context; + this.fasterButMoreRam = fasterButMoreRam; } protected abstract Directory getDirectory() throws IOException; @@ -54,10 +71,10 @@ public abstract class DocValuesWriterBas } @Override - public DocValuesConsumer addValuesField(DocValues.Type valueType, FieldInfo field) throws IOException { + public DocValuesConsumer addValuesField(Type valueType, FieldInfo field) throws IOException { return Writer.create(valueType, docValuesId(segmentName, field.number), - getDirectory(), getComparator(), bytesUsed, context); + getDirectory(), getComparator(), bytesUsed, context, fasterButMoreRam); } public static String docValuesId(String segmentsName, int fieldId) { Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene40/values/Bytes.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene40/values/Bytes.java?rev=1226460&r1=1226459&r2=1226460&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene40/values/Bytes.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene40/values/Bytes.java Mon Jan 2 16:26:10 2012 @@ -114,13 +114,17 @@ public final class Bytes { * {@link Writer}. A call to {@link Writer#finish(int)} will release * all internally used resources and frees the memory tracking * reference. - * @param context + * @param fasterButMoreRam whether packed ints for docvalues should be optimized for speed by rounding up the bytes + * used for a value to either 8, 16, 32 or 64 bytes. This option is only applicable for + * docvalues of type {@link Type#BYTES_FIXED_SORTED} and {@link Type#BYTES_VAR_SORTED}. + * @param context I/O Context * @return a new {@link Writer} instance * @throws IOException * if the files for the writer can not be created. */ public static Writer getWriter(Directory dir, String id, Mode mode, - boolean fixedSize, Comparator sortComparator, Counter bytesUsed, IOContext context) + boolean fixedSize, Comparator sortComparator, + Counter bytesUsed, IOContext context, boolean fasterButMoreRam) throws IOException { // TODO -- i shouldn't have to specify fixed? can // track itself & do the write thing at write time? @@ -134,7 +138,7 @@ public final class Bytes { } else if (mode == Mode.DEREF) { return new FixedDerefBytesImpl.Writer(dir, id, bytesUsed, context); } else if (mode == Mode.SORTED) { - return new FixedSortedBytesImpl.Writer(dir, id, sortComparator, bytesUsed, context); + return new FixedSortedBytesImpl.Writer(dir, id, sortComparator, bytesUsed, context, fasterButMoreRam); } } else { if (mode == Mode.STRAIGHT) { @@ -142,7 +146,7 @@ public final class Bytes { } else if (mode == Mode.DEREF) { return new VarDerefBytesImpl.Writer(dir, id, bytesUsed, context); } else if (mode == Mode.SORTED) { - return new VarSortedBytesImpl.Writer(dir, id, sortComparator, bytesUsed, context); + return new VarSortedBytesImpl.Writer(dir, id, sortComparator, bytesUsed, context, fasterButMoreRam); } } @@ -386,23 +390,32 @@ public final class Bytes { protected int lastDocId = -1; protected int[] docToEntry; protected final BytesRefHash hash; + protected final boolean fasterButMoreRam; protected long maxBytes = 0; protected DerefBytesWriterBase(Directory dir, String id, String codecName, int codecVersion, Counter bytesUsed, IOContext context) throws IOException { this(dir, id, codecName, codecVersion, new DirectTrackingAllocator( - ByteBlockPool.BYTE_BLOCK_SIZE, bytesUsed), bytesUsed, context); + ByteBlockPool.BYTE_BLOCK_SIZE, bytesUsed), bytesUsed, context, false); + } + + protected DerefBytesWriterBase(Directory dir, String id, String codecName, + int codecVersion, Counter bytesUsed, IOContext context, boolean fasterButMoreRam) + throws IOException { + this(dir, id, codecName, codecVersion, new DirectTrackingAllocator( + ByteBlockPool.BYTE_BLOCK_SIZE, bytesUsed), bytesUsed, context, fasterButMoreRam); } protected DerefBytesWriterBase(Directory dir, String id, String codecName, int codecVersion, Allocator allocator, - Counter bytesUsed, IOContext context) throws IOException { + Counter bytesUsed, IOContext context, boolean fasterButMoreRam) throws IOException { super(dir, id, codecName, codecVersion, bytesUsed, context); hash = new BytesRefHash(new ByteBlockPool(allocator), BytesRefHash.DEFAULT_CAPACITY, new TrackingDirectBytesStartArray( BytesRefHash.DEFAULT_CAPACITY, bytesUsed)); docToEntry = new int[1]; bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_INT); + this.fasterButMoreRam = fasterButMoreRam; } protected static int writePrefixLength(DataOutput datOut, BytesRef bytes) @@ -499,7 +512,7 @@ public final class Bytes { protected void writeIndex(IndexOutput idxOut, int docCount, long maxValue, int[] addresses, int[] toEntry) throws IOException { final PackedInts.Writer w = PackedInts.getWriter(idxOut, docCount, - PackedInts.bitsRequired(maxValue)); + bitsRequired(maxValue)); final int limit = docCount > docToEntry.length ? docToEntry.length : docCount; assert toEntry.length >= limit -1; @@ -523,7 +536,7 @@ public final class Bytes { protected void writeIndex(IndexOutput idxOut, int docCount, long maxValue, long[] addresses, int[] toEntry) throws IOException { final PackedInts.Writer w = PackedInts.getWriter(idxOut, docCount, - PackedInts.bitsRequired(maxValue)); + bitsRequired(maxValue)); final int limit = docCount > docToEntry.length ? docToEntry.length : docCount; assert toEntry.length >= limit -1; @@ -543,6 +556,11 @@ public final class Bytes { } w.finish(); } + + protected int bitsRequired(long maxValue){ + return fasterButMoreRam ? + PackedInts.getNextFixedSize(PackedInts.bitsRequired(maxValue)) : PackedInts.bitsRequired(maxValue); + } } Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene40/values/FixedSortedBytesImpl.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene40/values/FixedSortedBytesImpl.java?rev=1226460&r1=1226459&r2=1226460&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene40/values/FixedSortedBytesImpl.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene40/values/FixedSortedBytesImpl.java Mon Jan 2 16:26:10 2012 @@ -56,8 +56,8 @@ class FixedSortedBytesImpl { private final Comparator comp; public Writer(Directory dir, String id, Comparator comp, - Counter bytesUsed, IOContext context) throws IOException { - super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context); + Counter bytesUsed, IOContext context, boolean fasterButMoreRam) throws IOException { + super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context, fasterButMoreRam); this.comp = comp; } Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene40/values/VarSortedBytesImpl.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene40/values/VarSortedBytesImpl.java?rev=1226460&r1=1226459&r2=1226460&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene40/values/VarSortedBytesImpl.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene40/values/VarSortedBytesImpl.java Mon Jan 2 16:26:10 2012 @@ -57,8 +57,8 @@ final class VarSortedBytesImpl { private final Comparator comp; public Writer(Directory dir, String id, Comparator comp, - Counter bytesUsed, IOContext context) throws IOException { - super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context); + Counter bytesUsed, IOContext context, boolean fasterButMoreRam) throws IOException { + super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context, fasterButMoreRam); this.comp = comp; size = 0; } @@ -123,7 +123,7 @@ final class VarSortedBytesImpl { // total bytes of data idxOut.writeLong(maxBytes); PackedInts.Writer offsetWriter = PackedInts.getWriter(idxOut, count+1, - PackedInts.bitsRequired(maxBytes)); + bitsRequired(maxBytes)); // first dump bytes data, recording index & write offset as // we go final BytesRef spare = new BytesRef(); Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene40/values/Writer.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene40/values/Writer.java?rev=1226460&r1=1226459&r2=1226460&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene40/values/Writer.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/codecs/lucene40/values/Writer.java Mon Jan 2 16:26:10 2012 @@ -175,11 +175,14 @@ public abstract class Writer extends Doc * the {@link Directory} to create the files from. * @param bytesUsed * a byte-usage tracking reference + * @param fasterButMoreRam Whether the space used for packed ints should be rounded up for higher lookup performance. + * Currently this parameter only applies for types {@link Type#BYTES_VAR_SORTED} + * and {@link Type#BYTES_FIXED_SORTED}. * @return a new {@link Writer} instance for the given {@link Type} * @throws IOException */ public static Writer create(Type type, String id, Directory directory, - Comparator comp, Counter bytesUsed, IOContext context) throws IOException { + Comparator comp, Counter bytesUsed, IOContext context, boolean fasterButMoreRam) throws IOException { if (comp == null) { comp = BytesRef.getUTF8SortedAsUnicodeComparator(); } @@ -196,22 +199,22 @@ public abstract class Writer extends Doc return Floats.getWriter(directory, id, bytesUsed, context, type); case BYTES_FIXED_STRAIGHT: return Bytes.getWriter(directory, id, Bytes.Mode.STRAIGHT, true, comp, - bytesUsed, context); + bytesUsed, context, fasterButMoreRam); case BYTES_FIXED_DEREF: return Bytes.getWriter(directory, id, Bytes.Mode.DEREF, true, comp, - bytesUsed, context); + bytesUsed, context, fasterButMoreRam); case BYTES_FIXED_SORTED: return Bytes.getWriter(directory, id, Bytes.Mode.SORTED, true, comp, - bytesUsed, context); + bytesUsed, context, fasterButMoreRam); case BYTES_VAR_STRAIGHT: return Bytes.getWriter(directory, id, Bytes.Mode.STRAIGHT, false, comp, - bytesUsed, context); + bytesUsed, context, fasterButMoreRam); case BYTES_VAR_DEREF: return Bytes.getWriter(directory, id, Bytes.Mode.DEREF, false, comp, - bytesUsed, context); + bytesUsed, context, fasterButMoreRam); case BYTES_VAR_SORTED: return Bytes.getWriter(directory, id, Bytes.Mode.SORTED, false, comp, - bytesUsed, context); + bytesUsed, context, fasterButMoreRam); default: throw new IllegalArgumentException("Unknown Values: " + type); } Modified: lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/analysis/MockTokenizer.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/analysis/MockTokenizer.java?rev=1226460&r1=1226459&r2=1226460&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/analysis/MockTokenizer.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/analysis/MockTokenizer.java Mon Jan 2 16:26:10 2012 @@ -48,9 +48,9 @@ public class MockTokenizer extends Token public static final CharacterRunAutomaton KEYWORD = new CharacterRunAutomaton(new RegExp(".*").toAutomaton()); /** Acts like LetterTokenizer. */ - // the ugly regex below is Unicode 5.2 [:Letter:] + // the ugly regex below is incomplete Unicode 5.2 [:Letter:] public static final CharacterRunAutomaton SIMPLE = - new CharacterRunAutomaton(new RegExp("[A-Za-zªµºÀ-ÖØ-öø-ˁˆ-ˑˠ-ˤˬˮͰ-ʹͶͷͺ-ͽΆΈ-ΊΌΎ-ΡΣ-ϵϷ-ҁҊ-Ô¥Ô±-Ֆՙա-ևא-תװ-ײء-يٮٯٱ-ۓەۥۦۮۯۺ-ۼۿܐܒ-ܯݍ-ޥޱߊ-ߪߴߵߺࠀ-ࠕࠚࠤࠨऄ-हऽॐक़-ॡॱॲॹ-ॿঅ-ঌএঐও-নপ-রলশ-হঽৎড়ঢ়য়-ৡৰৱਅ-ਊਏਐਓ-ਨਪ-ਰਲਲ਼ਵਸ਼ਸਹਖ਼-ੜਫ਼ੲ-ੴઅ-ઍએ-ઑઓ-નપ-રલળવ-હઽૐૠૡଅ-ଌଏଐଓ-ନପ-ରଲଳଵ-ହଽଡ଼ଢ଼ୟ-ୡୱஃஅ-ஊஎ-ஐஒ-கஙசஜஞடணதந-ப஠®-ஹௐఅ-ఌఎ-ఐఒ-నప-ళవ-హఽౘౙౠౡಅ-ಌಎ-ಐಒ-ನಪ-ಳವ-ಹಽೞೠೡഅ-ഌഎ-ഐഒ-നപ-ഹഽൠൡൺ-ൿඅ-ඖක-නඳ-රලව-ෆก-ะาำเ-ๆກຂຄງຈຊຍດ-ທນ-ຟມ-ຣລວສຫອ-ະາຳຽເ-ໄໆໜໝༀཀ-ཇཉ-ཬྈ-ྋက-ဪဿၐ-ၕၚ-ၝၡၥၦၮ-ၰၵ-ႁႎႠ-Ⴥა-ჺჼᄀ-ቈቊ-ቍቐ-ቖቘቚ-ቝበ-ኈኊ-ኍነ-ኰኲ-ኵኸ-ኾዀዂ-ዅወ-ዖዘ-ጐጒ-ጕጘ-ፚᎀ-ᎏᎠ-Ᏼᐁ-ᙬᙯ-ᙿᚁ-ᚚᚠ-ᛪ ᜀ-ᜌᜎ-ᜑᜠ-ᜱᝀ-ᝑᝠ-ᝬᝮ-ᝰក-ឳៗៜᠠ-ᡷᢀ-ᢨᢪᢰ-ᣵᤀ-ᤜᥐ-ᥭᥰ-ᥴᦀ-ᦫᧁ-ᧇᨀ-ᨖᨠ-ᩔᪧᬅ-ᬳᭅ-ᭋᮃ-ᮠᮮᮯᰀ-ᰣᱍ-ᱏᱚ-ᱽᳩ-ᳬᳮ-ᳱᴀ-ᶿḀ-ἕἘ-Ἕἠ-ὅὈ-Ὅὐ-ὗὙὛὝὟ-ώᾀ-ᾴᾶ-ᾼιῂ-ῄῆ-ῌῐ-ΐῖ-Ίῠ-Ῥῲ-ῴῶ-ῼⁱⁿₐ-ₔℂℇℊ-ℓℕℙ-ℝℤΩℨK-ℭℯ-ℹℼ-ℿⅅ-ⅉⅎↃↄⰀ-Ⱞⰰ-ⱞⱠ-ⳤⳫ-ⳮⴀ-ⴥⴰ-ⵥⵯⶀ-ⶖⶠ-ⶦⶨ-ⶮⶰ-ⶶⶸ-ⶾⷀ-ⷆⷈ-ⷎⷐ-ⷖⷘ-ⷞⸯ々〆〱-〵〻� �€¼ã-ゖゝ-ゟァ-ヺー-ヿㄅ-ㄭㄱ-ㆎㆠ-ㆷㇰ-ㇿ㐀-䶵一-鿋ꀀ-ꒌꓐ-ꓽꔀ-ꘌꘐ-ꘟꘪꘫꙀ-ꙟꙢ-ꙮꙿ-ꚗꚠ-ꛥꜗ-ꜟꜢ-ꞈꞋꞌꟻ-ꠁꠃ-ꠅꠇ-ꠊꠌ-ꠢꡀ-ꡳꢂ-ꢳꣲ-ꣷꣻꤊ-ꤥꤰ-ꥆꥠ-ꥼꦄ-ꦲꧏꨀ-ꨨꩀ-ꩂꩄ-ꩋꩠ-ꩶꩺꪀ-ꪯꪱꪵꪶꪹ-ꪽꫀꫂꫛ-ꫝꯀ-ꯢ가-힣ힰ-ퟆퟋ-ퟻ豈-鶴侮-ï©­ï©°-龎ff-stﬓ-ﬗיִײַ-ﬨשׁ-זּטּ-לּמּנּסּףּפּצּ-ﮱﯓ-ﴽﵐ-ﶏﶒ-ﷇﷰ-ﷻﹰ-ﹴﹶ-ﻼＡ-Ｚａ-ｚｦ-ﾾￂ-ￇￊ-ￏￒ-ￗￚ-ￜ𐀀-𐀋𐀍-𐀦𐀨-� �€ºð€¼ð€½ð€¿-𐁍𐁐-𐁝𐂀-𐃺𐊀-𐊜𐊠-𐋐𐌀-𐌞𐌰-𐍀𐍂-𐍉𐎀-𐎝𐎠-𐏃𐏈-𐏏𐐀-𐒝𐠀-𐠅𐠈𐠊-𐠵𐠷𐠸𐠼𐠿-𐡕𐤀-𐤕𐤠-𐤹𐨀𐨐-𐨓𐨕-𐨗𐨙-𐨳𐩠-𐩼𐬀-𐬵𐭀-𐭕𐭠-𐭲𐰀-𐱈𑂃-𑂯𒀀-𒍮𓀀-𓐮𝐀-𝑔𝑖-𝒜𝒞𝒟𝒢𝒥𝒦𝒩-𝒬𝒮-𝒹𝒻𝒽-𝓃𝓅-𝔅𝔇-𝔊𝔍-𝔔𝔖-𝔜𝔞-𝔹𝔻-𝔾𝕀-𝕄𝕆𝕊-𝕐𝕒-𝚥𝚨-𝛀𝛂-𝛚𝛜-𝛺𝛼-𝜔𝜖-𝜴𝜶-𝝎𝝐-𝝮𝝰-𝞈𝞊-𝞨ð� �žª-𝟂𝟄-𝟋𠀀-𪛖𪜀-𫜴丽-𪘀]+").toAutomaton()); + new CharacterRunAutomaton(new RegExp("[A-Za-zªµºÀ-ÖØ-öø-Ｚ]+").toAutomaton()); private final CharacterRunAutomaton runAutomaton; private final boolean lowerCase; Modified: lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/search/CheckHits.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/search/CheckHits.java?rev=1226460&r1=1226459&r2=1226460&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/search/CheckHits.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/search/CheckHits.java Mon Jan 2 16:26:10 2012 @@ -28,6 +28,7 @@ import junit.framework.Assert; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReader.AtomicReaderContext; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.LuceneTestCase; public class CheckHits { @@ -177,7 +178,7 @@ public class CheckHits { Assert.assertEquals(query.toString(defaultFieldName), correct, actual); - QueryUtils.check(random, query,searcher); + QueryUtils.check(random, query,searcher, LuceneTestCase.rarely(random)); } /** Tests that a Hits has an expected order of documents */ Modified: lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/search/QueryUtils.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/search/QueryUtils.java?rev=1226460&r1=1226459&r2=1226460&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/search/QueryUtils.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/search/QueryUtils.java Mon Jan 2 16:26:10 2012 @@ -105,7 +105,7 @@ public class QueryUtils { public static void check(Random random, Query q1, IndexSearcher s) { check(random, q1, s, true); } - private static void check(Random random, Query q1, IndexSearcher s, boolean wrap) { + public static void check(Random random, Query q1, IndexSearcher s, boolean wrap) { try { check(q1); if (s!=null) { Modified: lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java?rev=1226460&r1=1226459&r2=1226460&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java Mon Jan 2 16:26:10 2012 @@ -156,7 +156,10 @@ public class MockDirectoryWrapper extend if (crashed) throw new IOException("cannot sync after crash"); unSyncedFiles.removeAll(names); - delegate.sync(names); + if (LuceneTestCase.rarely(randomState) || delegate instanceof NRTCachingDirectory) { + // don't wear out our hardware so much in tests. + delegate.sync(names); + } } @Override Modified: lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCase.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCase.java?rev=1226460&r1=1226459&r2=1226460&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCase.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCase.java Mon Jan 2 16:26:10 2012 @@ -686,7 +686,7 @@ public abstract class LuceneTestCase ext } } - private final static int THREAD_STOP_GRACE_MSEC = 50; + private final static int THREAD_STOP_GRACE_MSEC = 10; // jvm-wide list of 'rogue threads' we found, so they only get reported once. private final static IdentityHashMap rogueThreads = new IdentityHashMap(); @@ -898,10 +898,10 @@ public abstract class LuceneTestCase ext if (r.nextBoolean()) { if (rarely(r)) { // crazy value - c.setMaxBufferedDocs(_TestUtil.nextInt(r, 2, 7)); + c.setMaxBufferedDocs(_TestUtil.nextInt(r, 2, 15)); } else { // reasonable value - c.setMaxBufferedDocs(_TestUtil.nextInt(r, 8, 1000)); + c.setMaxBufferedDocs(_TestUtil.nextInt(r, 16, 1000)); } } if (r.nextBoolean()) { @@ -914,15 +914,21 @@ public abstract class LuceneTestCase ext } } if (r.nextBoolean()) { - c.setIndexerThreadPool(new ThreadAffinityDocumentsWriterThreadPool(_TestUtil.nextInt(r, 1, 20))); + if (rarely(r)) { + // crazy value + c.setIndexerThreadPool(new ThreadAffinityDocumentsWriterThreadPool(_TestUtil.nextInt(r, 5, 20))); + } else { + // reasonable value + c.setIndexerThreadPool(new ThreadAffinityDocumentsWriterThreadPool(_TestUtil.nextInt(r, 1, 4))); + } } - if (r.nextBoolean()) { - c.setMergePolicy(newTieredMergePolicy()); + if (rarely(r)) { + c.setMergePolicy(new MockRandomMergePolicy(r)); } else if (r.nextBoolean()) { - c.setMergePolicy(newLogMergePolicy()); + c.setMergePolicy(newTieredMergePolicy()); } else { - c.setMergePolicy(new MockRandomMergePolicy(r)); + c.setMergePolicy(newLogMergePolicy()); } c.setReaderPooling(r.nextBoolean()); @@ -943,9 +949,9 @@ public abstract class LuceneTestCase ext logmp.setUseCompoundFile(r.nextBoolean()); logmp.setCalibrateSizeByDeletes(r.nextBoolean()); if (rarely(r)) { - logmp.setMergeFactor(_TestUtil.nextInt(r, 2, 4)); + logmp.setMergeFactor(_TestUtil.nextInt(r, 2, 9)); } else { - logmp.setMergeFactor(_TestUtil.nextInt(r, 5, 50)); + logmp.setMergeFactor(_TestUtil.nextInt(r, 10, 50)); } return logmp; } @@ -953,16 +959,24 @@ public abstract class LuceneTestCase ext public static TieredMergePolicy newTieredMergePolicy(Random r) { TieredMergePolicy tmp = new TieredMergePolicy(); if (rarely(r)) { - tmp.setMaxMergeAtOnce(_TestUtil.nextInt(r, 2, 4)); - tmp.setMaxMergeAtOnceExplicit(_TestUtil.nextInt(r, 2, 4)); + tmp.setMaxMergeAtOnce(_TestUtil.nextInt(r, 2, 9)); + tmp.setMaxMergeAtOnceExplicit(_TestUtil.nextInt(r, 2, 9)); } else { - tmp.setMaxMergeAtOnce(_TestUtil.nextInt(r, 5, 50)); - tmp.setMaxMergeAtOnceExplicit(_TestUtil.nextInt(r, 5, 50)); + tmp.setMaxMergeAtOnce(_TestUtil.nextInt(r, 10, 50)); + tmp.setMaxMergeAtOnceExplicit(_TestUtil.nextInt(r, 10, 50)); + } + if (rarely(r)) { + tmp.setMaxMergedSegmentMB(0.2 + r.nextDouble() * 2.0); + } else { + tmp.setMaxMergedSegmentMB(r.nextDouble() * 100); } - tmp.setMaxMergedSegmentMB(0.2 + r.nextDouble() * 2.0); tmp.setFloorSegmentMB(0.2 + r.nextDouble() * 2.0); tmp.setForceMergeDeletesPctAllowed(0.0 + r.nextDouble() * 30.0); - tmp.setSegmentsPerTier(_TestUtil.nextInt(r, 2, 20)); + if (rarely(r)) { + tmp.setSegmentsPerTier(_TestUtil.nextInt(r, 2, 20)); + } else { + tmp.setSegmentsPerTier(_TestUtil.nextInt(r, 10, 50)); + } tmp.setUseCompoundFile(r.nextBoolean()); tmp.setNoCFSRatio(0.1 + r.nextDouble()*0.8); tmp.setReclaimDeletesWeight(r.nextDouble()*4); @@ -1216,7 +1230,7 @@ public abstract class LuceneTestCase ext * with one that returns null for getSequentialSubReaders. */ public static IndexSearcher newSearcher(IndexReader r, boolean maybeWrap) throws IOException { - if (random.nextBoolean()) { + if (usually()) { if (maybeWrap && rarely()) { r = new SlowMultiReaderWrapper(r); } Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/codecs/lucene3x/TestSurrogates.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/codecs/lucene3x/TestSurrogates.java?rev=1226460&r1=1226459&r2=1226460&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/codecs/lucene3x/TestSurrogates.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/codecs/lucene3x/TestSurrogates.java Mon Jan 2 16:26:10 2012 @@ -302,7 +302,7 @@ public class TestSurrogates extends Luce for(int f=0;f uniqueTerms = new HashSet(); Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/codecs/lucene3x/TestTermInfosReaderIndex.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/codecs/lucene3x/TestTermInfosReaderIndex.java?rev=1226460&r1=1226459&r2=1226460&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/codecs/lucene3x/TestTermInfosReaderIndex.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/codecs/lucene3x/TestTermInfosReaderIndex.java Mon Jan 2 16:26:10 2012 @@ -55,32 +55,43 @@ import org.apache.lucene.store.IndexInpu import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; +import org.junit.AfterClass; import org.junit.BeforeClass; public class TestTermInfosReaderIndex extends LuceneTestCase { - private static final int NUMBER_OF_DOCUMENTS = 1000; - private static final int NUMBER_OF_FIELDS = 100; - private TermInfosReaderIndex index; - private Directory directory; - private SegmentTermEnum termEnum; - private int indexDivisor; - private int termIndexInterval; - private IndexReader reader; - private List sampleTerms; + private static int NUMBER_OF_DOCUMENTS; + private static int NUMBER_OF_FIELDS; + private static TermInfosReaderIndex index; + private static Directory directory; + private static SegmentTermEnum termEnum; + private static int indexDivisor; + private static int termIndexInterval; + private static IndexReader reader; + private static List sampleTerms; /** we will manually instantiate preflex-rw here */ @BeforeClass - public static void beforeClass() { + public static void beforeClass() throws Exception { LuceneTestCase.PREFLEX_IMPERSONATION_IS_ACTIVE = true; - } - - @Override - public void setUp() throws Exception { - super.setUp(); + IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT, + new MockAnalyzer(random, MockTokenizer.KEYWORD, false)); + + termIndexInterval = config.getTermIndexInterval(); indexDivisor = _TestUtil.nextInt(random, 1, 10); + NUMBER_OF_DOCUMENTS = atLeast(100); + NUMBER_OF_FIELDS = atLeast(Math.max(10, 3*termIndexInterval*indexDivisor/NUMBER_OF_DOCUMENTS)); + directory = newDirectory(); - termIndexInterval = populate(directory); + + config.setCodec(new PreFlexRWCodec()); + // turn off compound file, this test will open some index files directly. + LogMergePolicy mp = newLogMergePolicy(); + mp.setUseCompoundFile(false); + config.setMergePolicy(mp); + + + populate(directory, config); IndexReader r0 = IndexReader.open(directory); SegmentReader r = (SegmentReader) r0.getSequentialSubReaders()[0]; @@ -102,15 +113,18 @@ public class TestTermInfosReaderIndex ex reader = IndexReader.open(directory); sampleTerms = sample(reader,1000); - } - @Override - public void tearDown() throws Exception { + @AfterClass + public static void afterClass() throws Exception { termEnum.close(); reader.close(); directory.close(); - super.tearDown(); + termEnum = null; + reader = null; + directory = null; + index = null; + sampleTerms = null; } public void testSeekEnum() throws CorruptIndexException, IOException { @@ -141,7 +155,7 @@ public class TestTermInfosReaderIndex ex } } - private List sample(IndexReader reader, int size) throws IOException { + private static List sample(IndexReader reader, int size) throws IOException { List sample = new ArrayList(); Random random = new Random(); FieldsEnum fieldsEnum = MultiFields.getFields(reader).iterator(); @@ -166,22 +180,13 @@ public class TestTermInfosReaderIndex ex private Term findTermThatWouldBeAtIndex(SegmentTermEnum termEnum, int index) throws IOException { int termPosition = index * termIndexInterval * indexDivisor; for (int i = 0; i < termPosition; i++) { - if (!termEnum.next()) { - fail("Should not have run out of terms."); - } + // TODO: this test just uses random terms, so this is always possible + assumeTrue("ran out of terms.", termEnum.next()); } return termEnum.term(); } - private int populate(Directory directory) throws CorruptIndexException, LockObtainFailedException, IOException { - IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT, - new MockAnalyzer(random, MockTokenizer.KEYWORD, false)); - config.setCodec(new PreFlexRWCodec()); - // turn off compound file, this test will open some index files directly. - LogMergePolicy mp = newLogMergePolicy(); - mp.setUseCompoundFile(false); - config.setMergePolicy(mp); - + private static void populate(Directory directory, IndexWriterConfig config) throws CorruptIndexException, LockObtainFailedException, IOException { RandomIndexWriter writer = new RandomIndexWriter(random, directory, config); for (int i = 0; i < NUMBER_OF_DOCUMENTS; i++) { Document document = new Document(); @@ -192,10 +197,9 @@ public class TestTermInfosReaderIndex ex } writer.forceMerge(1); writer.close(); - return config.getTermIndexInterval(); } - private String getText() { + private static String getText() { return Long.toString(random.nextLong(),Character.MAX_RADIX); } } Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/codecs/lucene40/TestDocValues.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/codecs/lucene40/TestDocValues.java?rev=1226460&r1=1226459&r2=1226460&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/codecs/lucene40/TestDocValues.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/codecs/lucene40/TestDocValues.java Mon Jan 2 16:26:10 2012 @@ -63,7 +63,8 @@ public class TestDocValues extends Lucen Directory dir = newDirectory(); final Counter trackBytes = Counter.newCounter(); - Writer w = Bytes.getWriter(dir, "test", mode, fixedSize, COMP, trackBytes, newIOContext(random)); + Writer w = Bytes.getWriter(dir, "test", mode, fixedSize, COMP, trackBytes, newIOContext(random), + random.nextBoolean()); int maxDoc = 220; final String[] values = new String[maxDoc]; final int fixedLength = 1 + atLeast(50); Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/codecs/pulsing/Test10KPulsings.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/codecs/pulsing/Test10KPulsings.java?rev=1226460&r1=1226459&r2=1226460&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/codecs/pulsing/Test10KPulsings.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/codecs/pulsing/Test10KPulsings.java Mon Jan 2 16:26:10 2012 @@ -47,6 +47,7 @@ import org.apache.lucene.util._TestUtil; * * @lucene.experimental */ +@LuceneTestCase.Nightly public class Test10KPulsings extends LuceneTestCase { public void test10kPulsed() throws Exception { // we always run this test with pulsing codec. Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=1226460&r1=1226459&r2=1226460&view=diff ============================================================================== --- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (original) +++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Mon Jan 2 16:26:10 2012 @@ -23,7 +23,9 @@ import java.io.IOException; import java.io.PrintStream; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Random; import org.apache.lucene.analysis.MockAnalyzer; @@ -52,6 +54,8 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Constants; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; +import org.junit.AfterClass; +import org.junit.BeforeClass; /* Verify we can read the pre-4.0 file format, do searches @@ -94,7 +98,7 @@ public class TestBackwardsCompatibility } */ - final String[] oldNames = {"30.cfs", + final static String[] oldNames = {"30.cfs", "30.nocfs", "31.cfs", "31.nocfs", @@ -120,10 +124,34 @@ public class TestBackwardsCompatibility "29.nocfs", }; - final String[] oldSingleSegmentNames = {"31.optimized.cfs", + final static String[] oldSingleSegmentNames = {"31.optimized.cfs", "31.optimized.nocfs", }; + static Map oldIndexDirs; + + @BeforeClass + public static void beforeClass() throws Exception { + List names = new ArrayList(oldNames.length + oldSingleSegmentNames.length); + names.addAll(Arrays.asList(oldNames)); + names.addAll(Arrays.asList(oldSingleSegmentNames)); + oldIndexDirs = new HashMap(); + for (String name : names) { + File dir = _TestUtil.getTempDir(name); + File dataFile = new File(TestBackwardsCompatibility.class.getResource("index." + name + ".zip").toURI()); + _TestUtil.unzip(dataFile, dir); + oldIndexDirs.put(name, newFSDirectory(dir)); + } + } + + @AfterClass + public static void afterClass() throws Exception { + for (Directory d : oldIndexDirs.values()) { + d.close(); + } + oldIndexDirs = null; + } + /** This test checks that *only* IndexFormatTooOldExceptions are thrown when you open and operate on too old indexes! */ public void testUnsupportedOldIndexes() throws Exception { for(int i=0;i failure = new ArrayList(); - Thread t1 = new Thread() { - @Override - public void run() { - boolean done = false; - while(!done) { - for(int i=0;i<100;i++) { - try { - finalWriter.addDocument(doc); - } catch (AlreadyClosedException e) { - done = true; - break; - } catch (NullPointerException e) { - done = true; - break; - } catch (Throwable e) { - e.printStackTrace(System.out); - failure.add(e); - done = true; - break; - } - } - Thread.yield(); - } - - } - }; - - if (failure.size() > 0) { - throw failure.get(0); - } - - t1.start(); - - writer.close(false); - t1.join(); - - // Make sure reader can read - IndexReader reader = IndexReader.open(directory); - reader.close(); - - // Reopen - writer = new IndexWriter(directory, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND).setMergePolicy(newLogMergePolicy())); - } - writer.close(); - } - - directory.close(); - } - - // LUCENE-1084: test unlimited field length public void testUnlimitedMaxFieldLength() throws IOException { Directory dir = newDirectory(); @@ -1613,102 +1516,6 @@ public class TestIndexWriter extends Luc dir.close(); } - public void testRandomStoredFields() throws IOException { - Directory dir = newDirectory(); - Random rand = random; - RandomIndexWriter w = new RandomIndexWriter(rand, dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(_TestUtil.nextInt(rand, 5, 20))); - //w.w.setUseCompoundFile(false); - final int docCount = atLeast(200); - final int fieldCount = _TestUtil.nextInt(rand, 1, 5); - - final List fieldIDs = new ArrayList(); - - FieldType customType = new FieldType(TextField.TYPE_STORED); - customType.setTokenized(false); - Field idField = newField("id", "", customType); - - for(int i=0;i docs = new HashMap(); - - if (VERBOSE) { - System.out.println("TEST: build index docCount=" + docCount); - } - - FieldType customType2 = new FieldType(); - customType2.setStored(true); - for(int i=0;i Number every so often - Collections.shuffle(fieldIDs); - } - if (rand.nextInt(5) == 3 && i > 0) { - final String delID = ""+rand.nextInt(i); - if (VERBOSE) { - System.out.println("TEST: delete doc id=" + delID); - } - w.deleteDocuments(new Term("id", delID)); - docs.remove(delID); - } - } - - if (VERBOSE) { - System.out.println("TEST: " + docs.size() + " docs in index; now load fields"); - } - if (docs.size() > 0) { - String[] idsList = docs.keySet().toArray(new String[docs.size()]); - - for(int x=0;x<2;x++) { - IndexReader r = w.getReader(); - IndexSearcher s = newSearcher(r); - - if (VERBOSE) { - System.out.println("TEST: cycle x=" + x + " r=" + r); - } - - int num = atLeast(1000); - for(int iter=0;iter