Return-Path: X-Original-To: apmail-lucene-commits-archive@www.apache.org Delivered-To: apmail-lucene-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 6C85D17A26 for ; Sat, 8 Nov 2014 11:32:55 +0000 (UTC) Received: (qmail 68917 invoked by uid 500); 8 Nov 2014 11:32:55 -0000 Mailing-List: contact commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@lucene.apache.org Delivered-To: mailing list commits@lucene.apache.org Received: (qmail 68907 invoked by uid 99); 8 Nov 2014 11:32:55 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 08 Nov 2014 11:32:55 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 08 Nov 2014 11:32:50 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 0E5CE23888A6; Sat, 8 Nov 2014 11:32:30 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1637544 [1/6] - in /lucene/dev/branches/lucene6005: ./ lucene/ lucene/core/ lucene/core/src/java/org/apache/lucene/codecs/perfield/ lucene/core/src/java/org/apache/lucene/document/ lucene/core/src/java/org/apache/lucene/index/ lucene/core/... Date: Sat, 08 Nov 2014 11:32:23 -0000 To: commits@lucene.apache.org From: mikemccand@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20141108113230.0E5CE23888A6@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: mikemccand Date: Sat Nov 8 11:32:18 2014 New Revision: 1637544 URL: http://svn.apache.org/r1637544 Log: LUCENE-6005: merge trunk Added: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/TooComplexToDeterminizeException.java - copied unchanged from r1637540, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/automaton/TooComplexToDeterminizeException.java lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/TestBitUtil.java - copied unchanged from r1637540, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/TestBitUtil.java lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java - copied unchanged from r1637540, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/automaton/TestRegExp.java - copied unchanged from r1637540, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/automaton/TestRegExp.java lucene/dev/branches/lucene6005/lucene/queries/src/test/org/apache/lucene/queries/TestCustomScoreExplanations.java - copied unchanged from r1637540, lucene/dev/trunk/lucene/queries/src/test/org/apache/lucene/queries/TestCustomScoreExplanations.java lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/search/BaseExplanationTestCase.java - copied unchanged from r1637540, lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/search/BaseExplanationTestCase.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/ConfigOverlay.java - copied unchanged from r1637540, lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/ConfigOverlay.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/MapSerializable.java - copied unchanged from r1637540, lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/MapSerializable.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java - copied unchanged from r1637540, lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/core/TestConfigOverlay.java - copied unchanged from r1637540, lucene/dev/trunk/solr/core/src/test/org/apache/solr/core/TestConfigOverlay.java lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java - copied unchanged from r1637540, lucene/dev/trunk/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java - copied unchanged from r1637540, lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotSmallAdvancedTest.java - copied unchanged from r1637540, lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotSmallAdvancedTest.java lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotWhiteBoxTest.java - copied unchanged from r1637540, lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotWhiteBoxTest.java lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/handler/component/FacetPivotSmallTest.java - copied unchanged from r1637540, lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/component/FacetPivotSmallTest.java Removed: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/BroadWord.java lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestExplanations.java lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/TestBroadWord.java Modified: lucene/dev/branches/lucene6005/ (props changed) lucene/dev/branches/lucene6005/lucene/ (props changed) lucene/dev/branches/lucene6005/lucene/CHANGES.txt (contents, props changed) lucene/dev/branches/lucene6005/lucene/core/ (props changed) lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/BinaryDocValuesField.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/Document.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldType.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldTypes.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/NumericDocValuesField.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedDocValuesField.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesField.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesField.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/MultiFields.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SimpleMergedSegmentWarmer.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/TermsHash.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/AutomatonQuery.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/FieldValueFilter.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/RegexpQuery.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/BitUtil.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/IOUtils.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/AutomatonProvider.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/ByteRunAutomaton.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/CharacterRunAutomaton.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/CompiledAutomaton.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/MinimizationOperations.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/Operations.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/RegExp.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/RunAutomaton.java lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/packed/EliasFanoDecoder.java lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/analysis/TestGraphTokenizers.java lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum2.java lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestAutomatonQuery.java lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestComplexExplanations.java lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestFieldValueFilter.java lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestRegexpQuery.java lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestSimpleExplanations.java lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadExplanations.java lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanExplanations.java lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/automaton/TestAutomaton.java lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/automaton/TestCompiledAutomaton.java lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/automaton/TestDeterminism.java lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/automaton/TestDeterminizeLexicon.java lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/automaton/TestLevenshteinAutomata.java lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/automaton/TestMinimize.java lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/automaton/TestOperations.java lucene/dev/branches/lucene6005/lucene/facet/ (props changed) lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/TestRandomSamplingFacetsCollector.java lucene/dev/branches/lucene6005/lucene/highlighter/ (props changed) lucene/dev/branches/lucene6005/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldQueryTest.java lucene/dev/branches/lucene6005/lucene/misc/ (props changed) lucene/dev/branches/lucene6005/lucene/misc/src/java/org/apache/lucene/uninverting/DocTermOrds.java lucene/dev/branches/lucene6005/lucene/misc/src/java/org/apache/lucene/uninverting/FieldCacheImpl.java lucene/dev/branches/lucene6005/lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java lucene/dev/branches/lucene6005/lucene/queries/ (props changed) lucene/dev/branches/lucene6005/lucene/queries/src/java/org/apache/lucene/queries/CustomScoreQuery.java lucene/dev/branches/lucene6005/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java lucene/dev/branches/lucene6005/lucene/queryparser/ (props changed) lucene/dev/branches/lucene6005/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java lucene/dev/branches/lucene6005/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/RegexpQueryNodeBuilder.java lucene/dev/branches/lucene6005/lucene/sandbox/ (props changed) lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java lucene/dev/branches/lucene6005/lucene/spatial/ (props changed) lucene/dev/branches/lucene6005/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java lucene/dev/branches/lucene6005/lucene/spatial/src/test/org/apache/lucene/spatial/bbox/TestBBoxStrategy.java lucene/dev/branches/lucene6005/lucene/suggest/ (props changed) lucene/dev/branches/lucene6005/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java lucene/dev/branches/lucene6005/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FuzzySuggester.java lucene/dev/branches/lucene6005/lucene/test-framework/ (props changed) lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/automaton/AutomatonTestUtil.java lucene/dev/branches/lucene6005/solr/ (props changed) lucene/dev/branches/lucene6005/solr/CHANGES.txt (contents, props changed) lucene/dev/branches/lucene6005/solr/bin/ (props changed) lucene/dev/branches/lucene6005/solr/bin/solr lucene/dev/branches/lucene6005/solr/bin/solr.cmd lucene/dev/branches/lucene6005/solr/core/ (props changed) lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/CloudUtil.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/Overseer.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/ZkController.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/Config.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/ConfigSolr.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/CoreContainer.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/InitParams.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/PluginInfo.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/RequestHandlers.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/SolrConfig.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/SolrCore.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/admin/EditFileRequestHandler.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/component/PivotFacetHelper.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/component/PivotFacetProcessor.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/component/PivotFacetValue.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/component/StatsField.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/response/SortingResponseWriter.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/rest/SolrConfigRestApi.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/schema/BBoxField.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/schema/FieldType.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/schema/IndexSchema.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchemaFactory.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/schema/ZkIndexSchemaReader.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/search/CacheConfig.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/search/mlt/CloudMLTQParser.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/util/CommandOperation.java lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/util/PivotListEntry.java lucene/dev/branches/lucene6005/solr/core/src/test-files/solr/collection1/conf/schema.xml lucene/dev/branches/lucene6005/solr/core/src/test-files/solr/collection1/conf/solrconfig-managed-schema.xml lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/cloud/TestCloudPivotFacet.java lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/cloud/TestReplicaProperties.java lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/core/TestInitParams.java lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotLargeTest.java lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotLongTailTest.java lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotSmallTest.java lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/response/TestSortingResponseWriter.java lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/rest/TestRestManager.java lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/rest/schema/TestBulkSchemaAPI.java lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/schema/IndexSchemaTest.java lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/schema/PrimitiveFieldTypeTest.java lucene/dev/branches/lucene6005/solr/core/src/test/org/apache/solr/search/mlt/CloudMLTQParserTest.java lucene/dev/branches/lucene6005/solr/solrj/ (props changed) lucene/dev/branches/lucene6005/solr/solrj/src/java/org/apache/solr/client/solrj/SolrQuery.java lucene/dev/branches/lucene6005/solr/solrj/src/java/org/apache/solr/client/solrj/response/FieldStatsInfo.java lucene/dev/branches/lucene6005/solr/solrj/src/java/org/apache/solr/client/solrj/response/PivotField.java lucene/dev/branches/lucene6005/solr/solrj/src/java/org/apache/solr/client/solrj/response/QueryResponse.java lucene/dev/branches/lucene6005/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java lucene/dev/branches/lucene6005/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java lucene/dev/branches/lucene6005/solr/solrj/src/java/org/apache/solr/common/util/JsonRecordReader.java lucene/dev/branches/lucene6005/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java lucene/dev/branches/lucene6005/solr/solrj/src/test/org/apache/solr/common/util/TestJsonRecordReader.java lucene/dev/branches/lucene6005/solr/test-framework/ (props changed) lucene/dev/branches/lucene6005/solr/test-framework/src/java/org/apache/solr/util/TestHarness.java Modified: lucene/dev/branches/lucene6005/lucene/CHANGES.txt URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/CHANGES.txt?rev=1637544&r1=1637543&r2=1637544&view=diff ============================================================================== --- lucene/dev/branches/lucene6005/lucene/CHANGES.txt (original) +++ lucene/dev/branches/lucene6005/lucene/CHANGES.txt Sat Nov 8 11:32:18 2014 @@ -201,6 +201,24 @@ Bug Fixes org.apache.lucene.index (Simon Willnauer, Robert Muir, Mike McCandless) +* LUCENE-6041: Remove sugar methods FieldInfo.isIndexed and + FieldInfo.hasDocValues. (Robert Muir, Mike McCandless) + +* LUCENE-6042: CustomScoreQuery explain was incorrect in some cases, + such as when nested inside a boolean query. (Denis Lantsman via Robert Muir) + +* LUCENE-6046: Add maxDeterminizedStates safety to determinize (which has + an exponential worst case) so that if it would create too many states, it + now throws an exception instead of exhausting CPU/RAM. (Nik + Everett via Mike McCandless) + +* LUCENE-6054: Allow repeating the empty automaton (Nik Everett via + Mike McCandless) + +* LUCENE-6049: Don't throw cryptic exception writing a segment when + the only docs in it had fields that hit non-aborting exceptions + during indexing but also had doc values. (Mike McCandless) + Documentation * LUCENE-5392: Add/improve analysis package documentation to reflect @@ -253,6 +271,9 @@ Optimizations * LUCENE-6030: Add norms patched compression for a small number of common values (Ryan Ernst) +* LUCENE-6040: Speed up EliasFanoDocIdSet through broadword bit selection. + (Paul Elschot) + Build * LUCENE-5909: Smoke tester now has better command line parsing and Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java?rev=1637544&r1=1637543&r2=1637544&view=diff ============================================================================== --- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java (original) +++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java Sat Nov 8 11:32:18 2014 @@ -30,6 +30,7 @@ import org.apache.lucene.codecs.DocValue import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SegmentReadState; @@ -251,7 +252,7 @@ public abstract class PerFieldDocValuesF try { // Read field name -> format name for (FieldInfo fi : readState.fieldInfos) { - if (fi.hasDocValues()) { + if (fi.getDocValuesType() != DocValuesType.NONE) { final String fieldName = fi.name; final String formatName = fi.getAttribute(PER_FIELD_FORMAT_KEY); if (formatName != null) { Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java?rev=1637544&r1=1637543&r2=1637544&view=diff ============================================================================== --- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java (original) +++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java Sat Nov 8 11:32:18 2014 @@ -36,6 +36,7 @@ import org.apache.lucene.codecs.FieldsPr import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.Fields; +import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.Terms; @@ -242,7 +243,7 @@ public abstract class PerFieldPostingsFo try { // Read field name -> format name for (FieldInfo fi : readState.fieldInfos) { - if (fi.isIndexed()) { + if (fi.getIndexOptions() != IndexOptions.NONE) { final String fieldName = fi.name; final String formatName = fi.getAttribute(PER_FIELD_FORMAT_KEY); if (formatName != null) { Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/BinaryDocValuesField.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/BinaryDocValuesField.java?rev=1637544&r1=1637543&r2=1637544&view=diff ============================================================================== --- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/BinaryDocValuesField.java (original) +++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/BinaryDocValuesField.java Sat Nov 8 11:32:18 2014 @@ -46,7 +46,7 @@ public class BinaryDocValuesField extend */ public static final FieldType TYPE = new FieldType(); static { - TYPE.setDocValueType(DocValuesType.BINARY); + TYPE.setDocValuesType(DocValuesType.BINARY); TYPE.freeze(); } Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/Document.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/Document.java?rev=1637544&r1=1637543&r2=1637544&view=diff ============================================================================== --- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/Document.java (original) +++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/Document.java Sat Nov 8 11:32:18 2014 @@ -270,7 +270,7 @@ public final class Document implements I return new FilterIterator(fields.iterator()) { @Override protected boolean predicateFunction(IndexableField field) { - return field.fieldType().stored() || field.fieldType().docValueType() != DocValuesType.NONE; + return field.fieldType().stored() || field.fieldType().docValuesType() != DocValuesType.NONE; } }; } Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldType.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldType.java?rev=1637544&r1=1637543&r2=1637544&view=diff ============================================================================== --- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldType.java (original) +++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldType.java Sat Nov 8 11:32:18 2014 @@ -54,7 +54,7 @@ public class FieldType implements Indexa private NumericType numericType; private boolean frozen; private int numericPrecisionStep = NumericUtils.PRECISION_STEP_DEFAULT; - private DocValuesType docValueType = DocValuesType.NONE; + private DocValuesType docValuesType = DocValuesType.NONE; /** * Create a new mutable FieldType with all of the properties from ref @@ -68,7 +68,7 @@ public class FieldType implements Indexa this.storeTermVectorPayloads = ref.storeTermVectorPayloads(); this.omitNorms = ref.omitNorms(); this.indexOptions = ref.indexOptions(); - this.docValueType = ref.docValueType(); + this.docValuesType = ref.docValuesType(); this.numericType = ref.numericType(); // Do not copy frozen! } @@ -377,12 +377,12 @@ public class FieldType implements Indexa result.append(numericPrecisionStep); } } - if (docValueType != DocValuesType.NONE) { + if (docValuesType != DocValuesType.NONE) { if (result.length() > 0) { result.append(","); } - result.append("docValueType="); - result.append(docValueType); + result.append("docValuesType="); + result.append(docValuesType); } return result.toString(); @@ -394,11 +394,11 @@ public class FieldType implements Indexa * {@inheritDoc} *

* The default is null (no docValues) - * @see #setDocValueType(DocValuesType) + * @see #setDocValuesType(DocValuesType) */ @Override - public DocValuesType docValueType() { - return docValueType; + public DocValuesType docValuesType() { + return docValuesType; } /** @@ -406,21 +406,21 @@ public class FieldType implements Indexa * @param type DocValues type, or null if no DocValues should be stored. * @throws IllegalStateException if this FieldType is frozen against * future modifications. - * @see #docValueType() + * @see #docValuesType() */ - public void setDocValueType(DocValuesType type) { + public void setDocValuesType(DocValuesType type) { checkIfFrozen(); if (type == null) { throw new NullPointerException("DocValuesType cannot be null"); } - docValueType = type; + docValuesType = type; } @Override public int hashCode() { final int prime = 31; int result = 1; - result = prime * result + ((docValueType == null) ? 0 : docValueType.hashCode()); + result = prime * result + ((docValuesType == null) ? 0 : docValuesType.hashCode()); result = prime * result + indexOptions.hashCode(); result = prime * result + numericPrecisionStep; result = prime * result + ((numericType == null) ? 0 : numericType.hashCode()); @@ -440,7 +440,7 @@ public class FieldType implements Indexa if (obj == null) return false; if (getClass() != obj.getClass()) return false; FieldType other = (FieldType) obj; - if (docValueType != other.docValueType) return false; + if (docValuesType != other.docValuesType) return false; if (indexOptions != other.indexOptions) return false; if (numericPrecisionStep != other.numericPrecisionStep) return false; if (numericType != other.numericType) return false; Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldTypes.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldTypes.java?rev=1637544&r1=1637543&r2=1637544&view=diff ============================================================================== --- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldTypes.java (original) +++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldTypes.java Sat Nov 8 11:32:18 2014 @@ -748,7 +748,7 @@ public class FieldTypes { } @Override - public DocValuesType docValueType() { + public DocValuesType docValuesType() { return docValuesType; } Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/NumericDocValuesField.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/NumericDocValuesField.java?rev=1637544&r1=1637543&r2=1637544&view=diff ============================================================================== --- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/NumericDocValuesField.java (original) +++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/NumericDocValuesField.java Sat Nov 8 11:32:18 2014 @@ -40,7 +40,7 @@ public class NumericDocValuesField exten */ public static final FieldType TYPE = new FieldType(); static { - TYPE.setDocValueType(DocValuesType.NUMERIC); + TYPE.setDocValuesType(DocValuesType.NUMERIC); TYPE.freeze(); } Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedDocValuesField.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedDocValuesField.java?rev=1637544&r1=1637543&r2=1637544&view=diff ============================================================================== --- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedDocValuesField.java (original) +++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedDocValuesField.java Sat Nov 8 11:32:18 2014 @@ -43,7 +43,7 @@ public class SortedDocValuesField extend */ public static final FieldType TYPE = new FieldType(); static { - TYPE.setDocValueType(DocValuesType.SORTED); + TYPE.setDocValuesType(DocValuesType.SORTED); TYPE.freeze(); } Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesField.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesField.java?rev=1637544&r1=1637543&r2=1637544&view=diff ============================================================================== --- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesField.java (original) +++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesField.java Sat Nov 8 11:32:18 2014 @@ -50,7 +50,7 @@ public class SortedNumericDocValuesField */ public static final FieldType TYPE = new FieldType(); static { - TYPE.setDocValueType(DocValuesType.SORTED_NUMERIC); + TYPE.setDocValuesType(DocValuesType.SORTED_NUMERIC); TYPE.freeze(); } Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesField.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesField.java?rev=1637544&r1=1637543&r2=1637544&view=diff ============================================================================== --- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesField.java (original) +++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesField.java Sat Nov 8 11:32:18 2014 @@ -44,7 +44,7 @@ public class SortedSetDocValuesField ext */ public static final FieldType TYPE = new FieldType(); static { - TYPE.setDocValueType(DocValuesType.SORTED_SET); + TYPE.setDocValuesType(DocValuesType.SORTED_SET); TYPE.freeze(); } Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1637544&r1=1637543&r2=1637544&view=diff ============================================================================== --- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (original) +++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java Sat Nov 8 11:32:18 2014 @@ -57,7 +57,6 @@ import org.apache.lucene.util.StringHelp import org.apache.lucene.util.Version; import org.apache.lucene.util.automaton.CompiledAutomaton; - /** * Basic tool and API to check the health of an index and * write a new segments file that removes reference to @@ -1127,7 +1126,7 @@ public class CheckIndex implements Close if (fieldInfo == null) { throw new RuntimeException("fieldsEnum inconsistent with fieldInfos, no fieldInfos for: " + field); } - if (!fieldInfo.isIndexed()) { + if (fieldInfo.getIndexOptions() == IndexOptions.NONE) { throw new RuntimeException("fieldsEnum inconsistent with fieldInfos, isIndexed == false for: " + field); } @@ -1769,7 +1768,7 @@ public class CheckIndex implements Close infoStream.print(" test: docvalues..........."); } for (FieldInfo fieldInfo : reader.getFieldInfos()) { - if (fieldInfo.hasDocValues()) { + if (fieldInfo.getDocValuesType() != DocValuesType.NONE) { status.totalValueFields++; checkDocValues(fieldInfo, reader, infoStream, status); } else { Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java?rev=1637544&r1=1637543&r2=1637544&view=diff ============================================================================== --- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java (original) +++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java Sat Nov 8 11:32:18 2014 @@ -129,7 +129,7 @@ final class DefaultIndexingChain extends PerField perField = fieldHash[i]; while (perField != null) { if (perField.docValuesWriter != null) { - if (perField.fieldInfo.hasDocValues() == false) { + if (perField.fieldInfo.getDocValuesType() == DocValuesType.NONE) { // BUG throw new AssertionError("segment=" + state.segmentInfo + ": field=\"" + perField.fieldInfo.name + "\" has no docValues but wrote them"); } @@ -142,7 +142,7 @@ final class DefaultIndexingChain extends perField.docValuesWriter.finish(docCount); perField.docValuesWriter.flush(state, dvConsumer); perField.docValuesWriter = null; - } else if (perField.fieldInfo.hasDocValues()) { + } else if (perField.fieldInfo.getDocValuesType() != DocValuesType.NONE) { // BUG throw new AssertionError("segment=" + state.segmentInfo + ": field=\"" + perField.fieldInfo.name + "\" has docValues but did not write them"); } @@ -199,7 +199,7 @@ final class DefaultIndexingChain extends // we must check the final value of omitNorms for the fieldinfo: it could have // changed for this field since the first time we added it. - if (fi.omitsNorms() == false && fi.isIndexed()) { + if (fi.omitsNorms() == false && fi.getIndexOptions() != IndexOptions.NONE) { assert perField.norms != null: "field=" + fi.name; perField.norms.finish(state.segmentInfo.getDocCount()); perField.norms.flush(state, normsConsumer); @@ -382,7 +382,7 @@ final class DefaultIndexingChain extends } } - DocValuesType dvType = fieldType.docValueType(); + DocValuesType dvType = fieldType.docValuesType(); if (dvType == null) { throw new NullPointerException("docValueType cannot be null (field: \"" + fieldName + "\")"); } @@ -419,9 +419,7 @@ final class DefaultIndexingChain extends * value */ private void indexDocValue(PerField fp, DocValuesType dvType, IndexableField field) throws IOException { - boolean hasDocValues = fp.fieldInfo.hasDocValues(); - - if (hasDocValues == false) { + if (fp.fieldInfo.getDocValuesType() == DocValuesType.NONE) { // This will throw an exc if the caller tried to // change the DV type for the field: fieldInfos.globalFieldNumbers.setDocValuesType(fp.fieldInfo.number, fp.fieldInfo.name, dvType); @@ -499,7 +497,11 @@ final class DefaultIndexingChain extends if (fp == null) { // First time we are seeing this field in this segment - FieldInfo fi = fieldInfos.addOrUpdate(name, fieldType); + FieldInfo fi = fieldInfos.getOrAdd(name); + // Messy: must set this here because e.g. FreqProxTermsWriterPerField looks at the initial + // IndexOptions to decide what arrays it must create). Then, we also must set it in + // PerField.invert to allow for later downgrading of the index options: + fi.setIndexOptions(fieldType.indexOptions()); fp = new PerField(fi, invert); fp.next = fieldHash[hashPos]; @@ -518,7 +520,10 @@ final class DefaultIndexingChain extends } } else { - fp.fieldInfo.update(fieldType); + // Messy: must set this here because e.g. FreqProxTermsWriterPerField looks at the initial + // IndexOptions to decide what arrays it must create). Then, we also must set it in + // PerField.invert to allow for later downgrading of the index options: + fp.fieldInfo.setIndexOptions(fieldType.indexOptions()); // NOTE: messy, but we must do this in case field was first seen w/o being // indexed, and now is seen again, this time being indexed: @@ -557,6 +562,8 @@ final class DefaultIndexingChain extends // reused TokenStream tokenStream; + IndexOptions indexOptions; + public PerField(FieldInfo fieldInfo, boolean invert) { this.fieldInfo = fieldInfo; similarity = docState.similarity; @@ -605,10 +612,17 @@ final class DefaultIndexingChain extends IndexableFieldType fieldType = field.fieldType(); + IndexOptions indexOptions = fieldType.indexOptions(); + fieldInfo.setIndexOptions(indexOptions); + + if (fieldType.omitNorms()) { + fieldInfo.setOmitsNorms(); + } + // only bother checking offsets if something will consume them. // nocommit can't we do this todo now? // TODO: after we fix analyzers, also check if termVectorOffsets will be indexed. - final boolean checkOffsets = fieldType.indexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS; + final boolean checkOffsets = indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS; /* * To assist people in tracking down problems in analysis components, we wish to write the field name to the infostream Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java?rev=1637544&r1=1637543&r2=1637544&view=diff ============================================================================== --- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java (original) +++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java Sat Nov 8 11:32:18 2014 @@ -108,10 +108,6 @@ public final class FieldInfo { return true; } - void update(IndexableFieldType ft) { - update(false, ft.omitNorms(), false, ft.indexOptions()); - } - // should only be called by FieldInfos#addOrUpdate void update(boolean storeTermVector, boolean omitNorms, boolean storePayloads, IndexOptions indexOptions) { if (indexOptions == null) { @@ -158,14 +154,24 @@ public final class FieldInfo { public IndexOptions getIndexOptions() { return indexOptions; } - - /** - * Returns true if this field has any docValues. - */ - public boolean hasDocValues() { - return docValuesType != DocValuesType.NONE; - } + /** Record the {@link IndexOptions} to use with this field. */ + public void setIndexOptions(IndexOptions newIndexOptions) { + if (indexOptions != newIndexOptions) { + if (indexOptions == IndexOptions.NONE) { + indexOptions = newIndexOptions; + } else if (newIndexOptions != IndexOptions.NONE) { + // downgrade + indexOptions = indexOptions.compareTo(newIndexOptions) < 0 ? indexOptions : newIndexOptions; + } + } + + if (indexOptions == IndexOptions.NONE || indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { + // cannot store payloads if we don't store positions: + storePayloads = false; + } + } + /** * Returns {@link DocValuesType} of the docValues; this is * {@code DocValuesType.NONE} if the field has no docvalues. @@ -206,19 +212,20 @@ public final class FieldInfo { public boolean omitsNorms() { return omitNorms; } + + /** Omit norms for this field. */ + public void setOmitsNorms() { + if (indexOptions == IndexOptions.NONE) { + throw new IllegalStateException("cannot omit norms: this field is not indexed"); + } + omitNorms = true; + } /** * Returns true if this field actually has any norms. */ public boolean hasNorms() { - return isIndexed() && omitNorms == false; - } - - /** - * Returns true if this field is indexed ({@link #getIndexOptions} is not IndexOptions.NONE). - */ - public boolean isIndexed() { - return indexOptions != IndexOptions.NONE; + return indexOptions != IndexOptions.NONE && omitNorms == false; } /** Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java?rev=1637544&r1=1637543&r2=1637544&view=diff ============================================================================== --- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java (original) +++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java Sat Nov 8 11:32:18 2014 @@ -68,11 +68,11 @@ public class FieldInfos implements Itera } hasVectors |= info.hasVectors(); - hasProx |= info.isIndexed() && info.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0; - hasFreq |= info.isIndexed() && info.getIndexOptions() != IndexOptions.DOCS; - hasOffsets |= info.isIndexed() && info.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; + hasProx |= info.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0; + hasFreq |= info.getIndexOptions() != IndexOptions.DOCS; + hasOffsets |= info.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; hasNorms |= info.hasNorms(); - hasDocValues |= info.hasDocValues(); + hasDocValues |= info.getDocValuesType() != DocValuesType.NONE; hasPayloads |= info.hasPayloads(); } @@ -276,21 +276,26 @@ public class FieldInfos implements Itera add(fieldInfo); } } - - /** NOTE: this method does not carry over termVector - * the indexer chain must set these fields when they - * succeed in consuming the document, nor the DocValuesType */ - public FieldInfo addOrUpdate(String name, IndexableFieldType fieldType) { - // TODO: really, indexer shouldn't even call this - // method (it's only called from DocFieldProcessor); - // rather, each component in the chain should update - // what it "owns". EG fieldType.indexOptions() should - // be updated by maybe FreqProxTermsWriterPerField: - return addOrUpdateInternal(name, -1, false, - fieldType.omitNorms(), false, - fieldType.indexOptions(), DocValuesType.NONE); - } + /** Create a new field, or return existing one. */ + public FieldInfo getOrAdd(String name) { + FieldInfo fi = fieldInfo(name); + if (fi == null) { + // This field wasn't yet added to this in-RAM + // segment's FieldInfo, so now we get a global + // number for this field. If the field was seen + // before then we'll get the same name and number, + // else we'll allocate a new one: + final int fieldNumber = globalFieldNumbers.addOrGet(name, -1, DocValuesType.NONE); + fi = new FieldInfo(name, fieldNumber, false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, null); + assert !byName.containsKey(fi.name); + globalFieldNumbers.verifyConsistent(Integer.valueOf(fi.number), fi.name, DocValuesType.NONE); + byName.put(fi.name, fi); + } + + return fi; + } + private FieldInfo addOrUpdateInternal(String name, int preferredFieldNumber, boolean storeTermVector, boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValuesType docValues) { @@ -314,10 +319,10 @@ public class FieldInfos implements Itera if (docValues != DocValuesType.NONE) { // Only pay the synchronization cost if fi does not already have a DVType - boolean updateGlobal = !fi.hasDocValues(); + boolean updateGlobal = fi.getDocValuesType() == DocValuesType.NONE; if (updateGlobal) { // Must also update docValuesType map so it's - // aware of this field's DocValueType. This will throw IllegalArgumentException if + // aware of this field's DocValuesType. This will throw IllegalArgumentException if // an illegal type change was attempted. globalFieldNumbers.setDocValuesType(fi.number, name, docValues); } Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java?rev=1637544&r1=1637543&r2=1637544&view=diff ============================================================================== --- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java (original) +++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java Sat Nov 8 11:32:18 2014 @@ -89,7 +89,7 @@ final class FreqProxTermsWriter extends final FreqProxTermsWriterPerField perField = (FreqProxTermsWriterPerField) f; if (perField.bytesHash.size() > 0) { perField.sortPostings(); - assert perField.fieldInfo.isIndexed(); + assert perField.fieldInfo.getIndexOptions() != IndexOptions.NONE; allFields.add(perField); } } Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1637544&r1=1637543&r2=1637544&view=diff ============================================================================== --- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original) +++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Sat Nov 8 11:32:18 2014 @@ -1532,7 +1532,7 @@ public class IndexWriter implements Clos DocValuesUpdate[] dvUpdates = new DocValuesUpdate[updates.length]; for (int i = 0; i < updates.length; i++) { final Field f = updates[i]; - final DocValuesType dvType = f.fieldType().docValueType(); + final DocValuesType dvType = f.fieldType().docValuesType(); if (dvType == null) { throw new NullPointerException("DocValuesType cannot be null (field: \"" + f.name() + "\")"); } Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java?rev=1637544&r1=1637543&r2=1637544&view=diff ============================================================================== --- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java (original) +++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java Sat Nov 8 11:32:18 2014 @@ -84,5 +84,5 @@ public interface IndexableFieldType { * DocValues {@link DocValuesType}: how the field's value will be indexed * into docValues. */ - public DocValuesType docValueType(); + public DocValuesType docValuesType(); } Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/MultiFields.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/MultiFields.java?rev=1637544&r1=1637543&r2=1637544&view=diff ============================================================================== --- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/MultiFields.java (original) +++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/MultiFields.java Sat Nov 8 11:32:18 2014 @@ -19,11 +19,11 @@ package org.apache.lucene.index; import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Collection; -import java.util.HashSet; import java.util.concurrent.ConcurrentHashMap; import org.apache.lucene.util.Bits; @@ -271,7 +271,7 @@ public final class MultiFields extends F public static Collection getIndexedFields(IndexReader reader) { final Collection fields = new HashSet<>(); for(final FieldInfo fieldInfo : getMergedFieldInfos(reader)) { - if (fieldInfo.isIndexed()) { + if (fieldInfo.getIndexOptions() != IndexOptions.NONE) { fields.add(fieldInfo.name); } } Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java?rev=1637544&r1=1637543&r2=1637544&view=diff ============================================================================== --- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java (original) +++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java Sat Nov 8 11:32:18 2014 @@ -497,12 +497,12 @@ class ReadersAndUpdates { } // create new fields or update existing ones to have NumericDV type for (String f : dvUpdates.numericDVUpdates.keySet()) { - FieldInfo fieldInfo = builder.addOrUpdate(f, NumericDocValuesField.TYPE); + FieldInfo fieldInfo = builder.getOrAdd(f); fieldInfo.setDocValuesType(DocValuesType.NUMERIC); } // create new fields or update existing ones to have BinaryDV type for (String f : dvUpdates.binaryDVUpdates.keySet()) { - FieldInfo fieldInfo = builder.addOrUpdate(f, BinaryDocValuesField.TYPE); + FieldInfo fieldInfo = builder.getOrAdd(f); fieldInfo.setDocValuesType(DocValuesType.BINARY); } Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java?rev=1637544&r1=1637543&r2=1637544&view=diff ============================================================================== --- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java (original) +++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java Sat Nov 8 11:32:18 2014 @@ -53,7 +53,7 @@ class SegmentDocValuesProducer extends D try { DocValuesProducer baseProducer = null; for (FieldInfo fi : fieldInfos) { - if (!fi.hasDocValues()) { + if (fi.getDocValuesType() == DocValuesType.NONE) { continue; } long docValuesGen = fi.getDocValuesGen(); Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SimpleMergedSegmentWarmer.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SimpleMergedSegmentWarmer.java?rev=1637544&r1=1637543&r2=1637544&view=diff ============================================================================== --- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SimpleMergedSegmentWarmer.java (original) +++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SimpleMergedSegmentWarmer.java Sat Nov 8 11:32:18 2014 @@ -44,7 +44,7 @@ public class SimpleMergedSegmentWarmer e int docValuesCount = 0; int normsCount = 0; for (FieldInfo info : reader.getFieldInfos()) { - if (info.isIndexed()) { + if (info.getIndexOptions() != IndexOptions.NONE) { reader.terms(info.name); indexedCount++; @@ -54,7 +54,7 @@ public class SimpleMergedSegmentWarmer e } } - if (info.hasDocValues()) { + if (info.getDocValuesType() != DocValuesType.NONE) { switch(info.getDocValuesType()) { case NUMERIC: reader.getNumericDocValues(info.name); Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/TermsHash.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/TermsHash.java?rev=1637544&r1=1637543&r2=1637544&view=diff ============================================================================== --- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/TermsHash.java (original) +++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/TermsHash.java Sat Nov 8 11:32:18 2014 @@ -86,7 +86,7 @@ abstract class TermsHash { } } - abstract TermsHashPerField addField(FieldInvertState fieldInvertState, final FieldInfo fieldInfo); + abstract TermsHashPerField addField(FieldInvertState fieldInvertState, FieldInfo fieldInfo); void finishDocument() throws IOException { if (nextTermsHash != null) { Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/AutomatonQuery.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/AutomatonQuery.java?rev=1637544&r1=1637543&r2=1637544&view=diff ============================================================================== --- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/AutomatonQuery.java (original) +++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/AutomatonQuery.java Sat Nov 8 11:32:18 2014 @@ -26,6 +26,7 @@ import org.apache.lucene.util.AttributeS import org.apache.lucene.util.ToStringUtils; import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.CompiledAutomaton; +import org.apache.lucene.util.automaton.Operations; /** * A {@link Query} that will match terms against a finite-state machine. @@ -61,10 +62,26 @@ public class AutomatonQuery extends Mult * match. */ public AutomatonQuery(final Term term, Automaton automaton) { + this(term, automaton, Operations.DEFAULT_MAX_DETERMINIZED_STATES); + } + + /** + * Create a new AutomatonQuery from an {@link Automaton}. + * + * @param term Term containing field and possibly some pattern structure. The + * term text is ignored. + * @param automaton Automaton to run, terms that are accepted are considered a + * match. + * @param maxDeterminizedStates maximum number of states in the resulting + * automata. If the automata would need more than this many states + * TooComplextToDeterminizeException is thrown. Higher number require more + * space but can process more complex automata. + */ + public AutomatonQuery(final Term term, Automaton automaton, int maxDeterminizedStates) { super(term.field()); this.term = term; this.automaton = automaton; - this.compiled = new CompiledAutomaton(automaton); + this.compiled = new CompiledAutomaton(automaton, null, true, maxDeterminizedStates); } @Override Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/FieldValueFilter.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/FieldValueFilter.java?rev=1637544&r1=1637543&r2=1637544&view=diff ============================================================================== --- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/FieldValueFilter.java (original) +++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/FieldValueFilter.java Sat Nov 8 11:32:18 2014 @@ -20,6 +20,8 @@ import java.io.IOException; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.DocValues; +import org.apache.lucene.util.BitDocIdSet; +import org.apache.lucene.util.BitSet; import org.apache.lucene.util.Bits; import org.apache.lucene.util.Bits.MatchAllBits; import org.apache.lucene.util.Bits.MatchNoBits; @@ -93,10 +95,10 @@ public class FieldValueFilter extends Fi if (docsWithField instanceof MatchNoBits) { return null; } - if (docsWithField instanceof DocIdSet) { + if (docsWithField instanceof BitSet) { // UweSays: this is always the case for our current impl - but who knows // :-) - return BitsFilteredDocIdSet.wrap((DocIdSet) docsWithField, acceptDocs); + return BitsFilteredDocIdSet.wrap(new BitDocIdSet((BitSet) docsWithField), acceptDocs); } return new DocValuesDocIdSet(context.reader().maxDoc(), acceptDocs) { @Override Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/RegexpQuery.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/RegexpQuery.java?rev=1637544&r1=1637543&r2=1637544&view=diff ============================================================================== --- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/RegexpQuery.java (original) +++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/RegexpQuery.java Sat Nov 8 11:32:18 2014 @@ -1,11 +1,5 @@ package org.apache.lucene.search; -import org.apache.lucene.index.Term; -import org.apache.lucene.util.ToStringUtils; -import org.apache.lucene.util.automaton.Automaton; -import org.apache.lucene.util.automaton.AutomatonProvider; -import org.apache.lucene.util.automaton.RegExp; - /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -23,6 +17,13 @@ import org.apache.lucene.util.automaton. * limitations under the License. */ +import org.apache.lucene.index.Term; +import org.apache.lucene.util.ToStringUtils; +import org.apache.lucene.util.automaton.Automaton; +import org.apache.lucene.util.automaton.AutomatonProvider; +import org.apache.lucene.util.automaton.Operations; +import org.apache.lucene.util.automaton.RegExp; + /** * A fast regular expression query based on the * {@link org.apache.lucene.util.automaton} package. @@ -75,18 +76,38 @@ public class RegexpQuery extends Automat * @param flags optional RegExp features from {@link RegExp} */ public RegexpQuery(Term term, int flags) { - this(term, flags, defaultProvider); + this(term, flags, defaultProvider, + Operations.DEFAULT_MAX_DETERMINIZED_STATES); } - + + /** + * Constructs a query for terms matching term. + * + * @param term regular expression. + * @param flags optional RegExp features from {@link RegExp} + * @param maxDeterminizedStates maximum number of states that compiling the + * automaton for the regexp can result in. Set higher to allow more complex + * queries and lower to prevent memory exhaustion. + */ + public RegexpQuery(Term term, int flags, int maxDeterminizedStates) { + this(term, flags, defaultProvider, maxDeterminizedStates); + } + /** * Constructs a query for terms matching term. * * @param term regular expression. * @param flags optional RegExp features from {@link RegExp} * @param provider custom AutomatonProvider for named automata + * @param maxDeterminizedStates maximum number of states that compiling the + * automaton for the regexp can result in. Set higher to allow more complex + * queries and lower to prevent memory exhaustion. */ - public RegexpQuery(Term term, int flags, AutomatonProvider provider) { - super(term, new RegExp(term.text(), flags).toAutomaton(provider)); + public RegexpQuery(Term term, int flags, AutomatonProvider provider, + int maxDeterminizedStates) { + super(term, + new RegExp(term.text(), flags).toAutomaton( + provider, maxDeterminizedStates), maxDeterminizedStates); } /** Prints a user-readable version of this query. */ Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java?rev=1637544&r1=1637543&r2=1637544&view=diff ============================================================================== --- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java (original) +++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java Sat Nov 8 11:32:18 2014 @@ -23,8 +23,8 @@ import java.util.List; import org.apache.lucene.index.Term; import org.apache.lucene.util.ToStringUtils; import org.apache.lucene.util.automaton.Automata; -import org.apache.lucene.util.automaton.Operations; import org.apache.lucene.util.automaton.Automaton; +import org.apache.lucene.util.automaton.Operations; /** Implements the wildcard search query. Supported wildcards are *, which * matches any character sequence (including the empty one), and ?, @@ -58,6 +58,17 @@ public class WildcardQuery extends Autom } /** + * Constructs a query for terms matching term. + * @param maxDeterminizedStates maximum number of states in the resulting + * automata. If the automata would need more than this many states + * TooComplextToDeterminizeException is thrown. Higher number require more + * space but can process more complex automata. + */ + public WildcardQuery(Term term, int maxDeterminizedStates) { + super(term, toAutomaton(term), maxDeterminizedStates); + } + + /** * Convert Lucene wildcard syntax into an automaton. * @lucene.internal */ Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/BitUtil.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/BitUtil.java?rev=1637544&r1=1637543&r2=1637544&view=diff ============================================================================== --- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/BitUtil.java (original) +++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/BitUtil.java Sat Nov 8 11:32:18 2014 @@ -212,4 +212,64 @@ public final class BitUtil { return ((l >>> 1) ^ -(l & 1)); } + + /** Select a 1-bit from a long. See also LUCENE-6040. + * @return The index of the r-th 1 bit in x. This bit must exist. + */ + public static int select(long x, int r) { + long s = x - ((x & 0xAAAAAAAAAAAAAAAAL) >>> 1); // pairwise bitsums + s = (s & 0x3333333333333333L) + ((s >>> 2) & 0x3333333333333333L); // nibblewise bitsums + s = ((s + (s >>> 4)) & 0x0F0F0F0F0F0F0F0FL) * L8_L; // bytewise bitsums, cumulative + + int b = (Long.numberOfTrailingZeros((s + psOverflow[r-1]) & (L8_L << 7)) >> 3) << 3; // bit position of byte with r-th 1 bit. + long l = r - (((s << 8) >>> b) & 0xFFL); // bit rank in byte at b + + // Select bit l from byte (x >>> b): + int selectIndex = (int) (((x >>> b) & 0xFFL) | ((l-1) << 8)); + int res = b + select256[selectIndex]; + return res; + } + + private final static long L8_L = 0x0101010101010101L; + + private static final long[] psOverflow = new long[64]; + static { + for (int s = 1; s <= 64; s++) { + psOverflow[s-1] = (128-s) * L8_L; + } + } + + private static final byte[] select256 = new byte[8 * 256]; + static { + for (int b = 0; b <= 0xFF; b++) { + for (int s = 1; s <= 8; s++) { + int byteIndex = b | ((s-1) << 8); + int bitIndex = selectNaive(b, s); + if (bitIndex < 0) { + bitIndex = 127; // positive as byte + } + assert bitIndex >= 0; + assert ((byte) bitIndex) >= 0; // non negative as byte, no need to mask the sign + select256[byteIndex] = (byte) bitIndex; + } + } + } + + /** + * Naive implementation of {@link #select(long,int)}, using {@link Long#numberOfTrailingZeros} repetitively. + * Works relatively fast for low ranks. + * @return The index of the r-th 1 bit in x, or -1 if no such bit exists. + */ + public static int selectNaive(long x, int r) { + assert r >= 1; + int s = -1; + while ((x != 0L) && (r > 0)) { + int ntz = Long.numberOfTrailingZeros(x); + x >>>= (ntz + 1); + s += (ntz + 1); + r -= 1; + } + int res = (r > 0) ? -1 : s; + return res; + } } Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/IOUtils.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/IOUtils.java?rev=1637544&r1=1637543&r2=1637544&view=diff ============================================================================== --- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/IOUtils.java (original) +++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/IOUtils.java Sat Nov 8 11:32:18 2014 @@ -37,6 +37,7 @@ import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.nio.file.attribute.BasicFileAttributes; import java.util.Arrays; +import java.util.Collection; import java.util.LinkedHashMap; import java.util.Map; @@ -215,7 +216,7 @@ public final class IOUtils { *

* Some of the files may be null, if so they are ignored. */ - public static void deleteFilesIgnoringExceptions(Iterable files) { + public static void deleteFilesIgnoringExceptions(Collection files) { for (Path name : files) { if (name != null) { try { @@ -249,7 +250,7 @@ public final class IOUtils { * * @param files files to delete */ - public static void deleteFilesIfExist(Iterable files) throws IOException { + public static void deleteFilesIfExist(Collection files) throws IOException { Throwable th = null; for (Path file : files) { Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/AutomatonProvider.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/AutomatonProvider.java?rev=1637544&r1=1637543&r2=1637544&view=diff ============================================================================== --- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/AutomatonProvider.java (original) +++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/AutomatonProvider.java Sat Nov 8 11:32:18 2014 @@ -33,7 +33,7 @@ import java.io.IOException; /** * Automaton provider for RegExp. - * {@link RegExp#toAutomaton(AutomatonProvider)} + * {@link RegExp#toAutomaton(AutomatonProvider,int)} * * @lucene.experimental */ Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/ByteRunAutomaton.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/ByteRunAutomaton.java?rev=1637544&r1=1637543&r2=1637544&view=diff ============================================================================== --- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/ByteRunAutomaton.java (original) +++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/ByteRunAutomaton.java Sat Nov 8 11:32:18 2014 @@ -24,12 +24,12 @@ public class ByteRunAutomaton extends Ru /** Converts incoming automaton to byte-based (UTF32ToUTF8) first */ public ByteRunAutomaton(Automaton a) { - this(a, false); + this(a, false, Operations.DEFAULT_MAX_DETERMINIZED_STATES); } /** expert: if isBinary is true, the input is already byte-based, else it's UTF-32 and we'll convert to UTF-8 */ - public ByteRunAutomaton(Automaton a, boolean isBinary) { - super(isBinary ? a : new UTF32ToUTF8().convert(a), 256, true); + public ByteRunAutomaton(Automaton a, boolean isBinary, int maxDeterminizedStates) { + super(isBinary ? a : new UTF32ToUTF8().convert(a), 256, true, maxDeterminizedStates); } /** Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/CharacterRunAutomaton.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/CharacterRunAutomaton.java?rev=1637544&r1=1637543&r2=1637544&view=diff ============================================================================== --- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/CharacterRunAutomaton.java (original) +++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/CharacterRunAutomaton.java Sat Nov 8 11:32:18 2014 @@ -21,10 +21,22 @@ package org.apache.lucene.util.automaton * Automaton representation for matching char[]. */ public class CharacterRunAutomaton extends RunAutomaton { - - /** Sole constructor. */ + /** + * Construct with a default number of maxDeterminizedStates. + */ public CharacterRunAutomaton(Automaton a) { - super(a, Character.MAX_CODE_POINT, false); + this(a, Operations.DEFAULT_MAX_DETERMINIZED_STATES); + } + + /** + * Construct specifying maxDeterminizedStates. + * @param a Automaton to match + * @param maxDeterminizedStates maximum number of states that the automataon + * can have once determinized. If more states are required to determinize + * it then a TooComplexToDeterminizeException is thrown. + */ + public CharacterRunAutomaton(Automaton a, int maxDeterminizedStates) { + super(a, Character.MAX_CODE_POINT, false, maxDeterminizedStates); } /** Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/CompiledAutomaton.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/CompiledAutomaton.java?rev=1637544&r1=1637543&r2=1637544&view=diff ============================================================================== --- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/CompiledAutomaton.java (original) +++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/CompiledAutomaton.java Sat Nov 8 11:32:18 2014 @@ -21,9 +21,9 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import org.apache.lucene.index.SingleTermsEnum; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.index.SingleTermsEnum; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; @@ -157,7 +157,8 @@ public class CompiledAutomaton { } } sinkState = foundState; - runAutomaton = new ByteRunAutomaton(automaton, true); + // It's safe to allow unlimited determinized here: + runAutomaton = new ByteRunAutomaton(automaton, true, Integer.MAX_VALUE); type = AUTOMATON_TYPE.RANGE; } @@ -177,7 +178,8 @@ public class CompiledAutomaton { automaton.addTransition(lastState, lastState, 0, 255); sinkState = lastState; automaton.finishState(); - runAutomaton = new ByteRunAutomaton(automaton, true); + // It's safe to allow unlimited determinized here: + runAutomaton = new ByteRunAutomaton(automaton, true, Integer.MAX_VALUE); commonSuffixRef = null; finite = false; minInclusive = false; @@ -190,7 +192,20 @@ public class CompiledAutomaton { * possibly expensive operations to determine if the automaton is one * the cases in {@link CompiledAutomaton.AUTOMATON_TYPE}. */ public CompiledAutomaton(Automaton automaton, Boolean finite, boolean simplify) { + this(automaton, finite, simplify, Operations.DEFAULT_MAX_DETERMINIZED_STATES); + } + + /** Create this. If finite is null, we use {@link Operations#isFinite} + * to determine whether it is finite. If simplify is true, we run + * possibly expensive operations to determine if the automaton is one + * the cases in {@link CompiledAutomaton.AUTOMATON_TYPE}. If simplify + * requires determinizing the autaomaton then only maxDeterminizedStates + * will be created. Any more than that will cause a + * TooComplexToDeterminizeException. + */ + public CompiledAutomaton(Automaton automaton, Boolean finite, boolean simplify, + int maxDeterminizedStates) { if (automaton.getNumStates() == 0) { automaton = new Automaton(); automaton.createState(); @@ -231,7 +246,7 @@ public class CompiledAutomaton { return; } else { - automaton = Operations.determinize(automaton); + automaton = Operations.determinize(automaton, maxDeterminizedStates); final String commonPrefix = Operations.getCommonPrefix(automaton); final String singleton; @@ -257,7 +272,7 @@ public class CompiledAutomaton { return; } else if (commonPrefix.length() > 0) { Automaton other = Operations.concatenate(Automata.makeString(commonPrefix), Automata.makeAnyString()); - other = Operations.determinize(other); + other = Operations.determinize(other, maxDeterminizedStates); assert Operations.hasDeadStates(other) == false; if (Operations.sameLanguage(automaton, other)) { // matches a constant prefix @@ -276,7 +291,8 @@ public class CompiledAutomaton { sinkState = lastState; automaton.finishState(); this.automaton = automaton; - runAutomaton = new ByteRunAutomaton(automaton, true); + // It's safe to allow unlimited determinized here: + runAutomaton = new ByteRunAutomaton(automaton, true, Integer.MAX_VALUE); this.finite = false; maxTerm = null; minInclusive = false; @@ -305,9 +321,9 @@ public class CompiledAutomaton { if (this.finite) { commonSuffixRef = null; } else { - commonSuffixRef = Operations.getCommonSuffixBytesRef(binary); + commonSuffixRef = Operations.getCommonSuffixBytesRef(binary, maxDeterminizedStates); } - runAutomaton = new ByteRunAutomaton(binary, true); + runAutomaton = new ByteRunAutomaton(binary, true, maxDeterminizedStates); this.automaton = runAutomaton.automaton; } Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/MinimizationOperations.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/MinimizationOperations.java?rev=1637544&r1=1637543&r2=1637544&view=diff ============================================================================== --- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/MinimizationOperations.java (original) +++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/automaton/MinimizationOperations.java Sat Nov 8 11:32:18 2014 @@ -29,8 +29,8 @@ package org.apache.lucene.util.automaton; -import java.util.BitSet; import java.util.ArrayList; +import java.util.BitSet; import java.util.HashSet; import java.util.LinkedList; @@ -45,21 +45,17 @@ final public class MinimizationOperation /** * Minimizes (and determinizes if not already deterministic) the given - * automaton. - */ - public static Automaton minimize(Automaton a) { - return minimizeHopcroft(a); - } - - /** - * Minimizes the given automaton using Hopcroft's algorithm. + * automaton using Hopcroft's algorighm. + * @param maxDeterminizedStates maximum number of states determinizing the + * automaton can result in. Set higher to allow more complex queries and + * lower to prevent memory exhaustion. */ - public static Automaton minimizeHopcroft(Automaton a) { + public static Automaton minimize(Automaton a, int maxDeterminizedStates) { if (a.getNumStates() == 0 || (a.isAccept(0) == false && a.getNumTransitions(0) == 0)) { // Fastmatch for common case return new Automaton(); } - a = Operations.determinize(a); + a = Operations.determinize(a, maxDeterminizedStates); //a.writeDot("adet"); if (a.getNumTransitions(0) == 1) { Transition t = new Transition();