Return-Path: X-Original-To: apmail-lucene-commits-archive@www.apache.org Delivered-To: apmail-lucene-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 9419D17275 for ; Wed, 2 Sep 2015 13:06:38 +0000 (UTC) Received: (qmail 50171 invoked by uid 500); 2 Sep 2015 13:06:35 -0000 Mailing-List: contact commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@lucene.apache.org Delivered-To: mailing list commits@lucene.apache.org Received: (qmail 49563 invoked by uid 99); 2 Sep 2015 13:06:35 -0000 Received: from eris.apache.org (HELO hades.apache.org) (140.211.11.105) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 02 Sep 2015 13:06:35 +0000 Received: from hades.apache.org (localhost [127.0.0.1]) by hades.apache.org (ASF Mail Server at hades.apache.org) with ESMTP id 16F30AC1C07 for ; Wed, 2 Sep 2015 13:06:35 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1700800 [13/24] - in /lucene/dev/branches/lucene6699: ./ dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/scripts/ lucene/ lucene/analysis/ lucene/analysis/common/ lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/... Date: Wed, 02 Sep 2015 13:06:22 -0000 To: commits@lucene.apache.org From: mikemccand@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20150902130635.16F30AC1C07@hades.apache.org> Modified: lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/schema11.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/schema11.xml?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/schema11.xml (original) +++ lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/schema11.xml Wed Sep 2 13:06:13 2015 @@ -336,46 +336,46 @@ valued. --> EXAMPLE: name="*_i" will match any field ending in _i (like myid_i, z_i) Longer patterns will be matched first. if equal size patterns both match, the first appearing in the schema will be used. --> - + - - - - - - - + + + + + + + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - - - - - - - - + + + + + + + + - - + + - - + + Modified: lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml (original) +++ lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml Wed Sep 2 13:06:13 2015 @@ -310,12 +310,12 @@ - + - + Another way to map multiple source fields to the same + destination field is to use the dynamic field syntax. + copyField also supports a maxChars to copy setting. --> + @@ -342,8 +342,8 @@ + removes stop words from case-insensitive "stopwords.txt" + (empty by default), and down cases. At query time only, it + also applies synonyms. --> @@ -492,11 +492,11 @@ words="stopwords.txt" /> - + - + --> @@ -507,23 +507,23 @@ words="stopwords.txt" /> - + - + --> @@ -574,7 +574,7 @@ + each token, to enable more efficient leading wildcard queries. --> @@ -649,10 +649,10 @@ a token of "foo|1.4" would be indexed as "foo" with a payload of 1.4f Attributes of the DelimitedPayloadTokenFilterFactory : "delimiter" - a one character delimiter. Default is | (pipe) - "encoder" - how to encode the following value into a playload - float -> org.apache.lucene.analysis.payloads.FloatEncoder, - integer -> o.a.l.a.p.IntegerEncoder - identity -> o.a.l.a.p.IdentityEncoder + "encoder" - how to encode the following value into a playload + float -> org.apache.lucene.analysis.payloads.FloatEncoder, + integer -> o.a.l.a.p.IntegerEncoder + identity -> o.a.l.a.p.IdentityEncoder Fully Qualified class name implementing PayloadEncoder, Encoder must have a no arg constructor. --> @@ -673,10 +673,10 @@ --> - + - + - + - + Modified: lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-components-name.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-components-name.xml?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-components-name.xml (original) +++ lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-components-name.xml Wed Sep 2 13:06:13 2015 @@ -41,8 +41,8 @@ class="solr.XMLResponseWriter" /> - true - + true + component1 Modified: lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-delpolicy2.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-delpolicy2.xml?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-delpolicy2.xml (original) +++ lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-delpolicy2.xml Wed Sep 2 13:06:13 2015 @@ -38,8 +38,8 @@ value1 - value2 - + value2 + Modified: lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-headers.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-headers.xml?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-headers.xml (original) +++ lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-headers.xml Wed Sep 2 13:06:13 2015 @@ -24,8 +24,8 @@ - - componentThatAddsHeader - + + componentThatAddsHeader + Modified: lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-nocache-with-delaying-searchcomponent.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-nocache-with-delaying-searchcomponent.xml?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-nocache-with-delaying-searchcomponent.xml (original) +++ lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-nocache-with-delaying-searchcomponent.xml Wed Sep 2 13:06:13 2015 @@ -27,9 +27,9 @@ class="org.apache.solr.search.DelayingSearchComponent"/> - - delayingSearchComponent - + + delayingSearchComponent + Modified: lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml (original) +++ lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml Wed Sep 2 13:06:13 2015 @@ -150,7 +150,7 @@ text - 2 + 2 Modified: lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-response-log-component.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-response-log-component.xml?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-response-log-component.xml (original) +++ lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-response-log-component.xml Wed Sep 2 13:06:13 2015 @@ -37,9 +37,9 @@ - - dismax - + + dismax + responselog Modified: lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-slave.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-slave.xml?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-slave.xml (original) +++ lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-slave.xml Wed Sep 2 13:06:13 2015 @@ -41,9 +41,9 @@ - - http://127.0.0.1:TEST_PORT/solr/collection1 - 00:00:01 + + http://127.0.0.1:TEST_PORT/solr/collection1 + 00:00:01 COMPRESSION Modified: lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-spellcheckcomponent.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-spellcheckcomponent.xml?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-spellcheckcomponent.xml (original) +++ lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-spellcheckcomponent.xml Wed Sep 2 13:06:13 2015 @@ -88,11 +88,11 @@ .29 - multipleFields - lowerfilt1and2 - spellcheckerMultipleFields - true - + multipleFields + lowerfilt1and2 + spellcheckerMultipleFields + true + jarowinkler @@ -156,13 +156,13 @@ - - dismax - lowerfilt1^1 - - - spellcheck - + + dismax + lowerfilt1^1 + + + spellcheck + Modified: lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-suggestercomponent.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-suggestercomponent.xml?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-suggestercomponent.xml (original) +++ lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-suggestercomponent.xml Wed Sep 2 13:06:13 2015 @@ -31,7 +31,7 @@ - + suggest_fuzzy_with_high_freq_dict FuzzyLookupFactory @@ -42,8 +42,8 @@ 0.0 - - + + suggest_fuzzy_file_based FuzzyLookupFactory @@ -53,7 +53,7 @@ true - + suggest_fuzzy_doc_dict FuzzyLookupFactory @@ -65,7 +65,7 @@ false - + suggest_fuzzy_doc_expr_dict DocumentExpressionDictionaryFactory Modified: lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-tolerant-search.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-tolerant-search.xml?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-tolerant-search.xml (original) +++ lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig-tolerant-search.xml Wed Sep 2 13:06:13 2015 @@ -1,52 +1,52 @@ + license agreements. See the NOTICE file distributed with this work for additional + information regarding copyright ownership. The ASF licenses this file to + You under the Apache License, Version 2.0 (the "License"); you may not use + this file except in compliance with the License. You may obtain a copy of + the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required + by applicable law or agreed to in writing, software distributed under the + License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS + OF ANY KIND, either express or implied. See the License for the specific + language governing permissions and limitations under the License. --> + a new test, feel free to add *new* items (plugins, config options, etc...) + as long as they don't break any existing tests. if you need to test something + esoteric please add a new "solrconfig-your-esoteric-purpose.xml" config file. + Note in particular that this test is used by MinimalSchemaTest so Anything + added to this file needs to work correctly even if there is now uniqueKey + or defaultSearch Field. --> - ${solr.data.dir:} + ${solr.data.dir:} - + - ${tests.luceneMatchVersion:LATEST} + ${tests.luceneMatchVersion:LATEST} - - - - - ${solr.commitwithin.softcommit:true} - - - - - - explicit - true - text - - - - - + + + + ${solr.commitwithin.softcommit:true} + + + + + + explicit + true + text + + + + + Modified: lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml (original) +++ lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml Wed Sep 2 13:06:13 2015 @@ -83,7 +83,7 @@ --> - ${solr.ulog.dir:} + ${solr.ulog.dir:} @@ -198,7 +198,7 @@ is not specified in the request. --> - true + true @@ -269,11 +269,11 @@ 10 - multipleFields - lowerfilt1and2 - spellcheckerMultipleFields - false - + multipleFields + lowerfilt1and2 + spellcheckerMultipleFields + false + jarowinkler @@ -377,13 +377,13 @@ - - dismax - lowerfilt1^1 - - - spellcheck - + + dismax + lowerfilt1^1 + + + spellcheck + Modified: lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/solr-50-all.xml URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/solr-50-all.xml?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/solr-50-all.xml (original) +++ lucene/dev/branches/lucene6699/solr/core/src/test-files/solr/solr-50-all.xml Wed Sep 2 13:06:13 2015 @@ -21,6 +21,7 @@ 11 ${coreRootDirectory:testCoreRootDirectory} testInfoHandler + testConfigSetsHandler testManagementPath testSharedLib ${shareSchema:true} Modified: lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java (original) +++ lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java Wed Sep 2 13:06:13 2015 @@ -710,7 +710,7 @@ public class BasicFunctionalityTest exte SolrQueryResponse rsp = new SolrQueryResponse(); core.execute(core.getRequestHandler(req.getParams().get(CommonParams.QT)), req, rsp); - DocList dl = ((ResultContext) rsp.getValues().get("response")).docs; + DocList dl = ((ResultContext) rsp.getValues().get("response")).getDocList(); StoredDocument d = req.getSearcher().doc(dl.iterator().nextDoc()); // ensure field in fl is not lazy assertFalse( ((Field) d.getField("test_hlt")).getClass().getSimpleName().equals("LazyField")); @@ -735,7 +735,7 @@ public class BasicFunctionalityTest exte SolrQueryResponse rsp = new SolrQueryResponse(); core.execute(core.getRequestHandler(req.getParams().get(CommonParams.QT)), req, rsp); - DocList dl = ((ResultContext) rsp.getValues().get("response")).docs; + DocList dl = ((ResultContext) rsp.getValues().get("response")).getDocList(); DocIterator di = dl.iterator(); StoredDocument d1 = req.getSearcher().doc(di.nextDoc()); StorableField[] values1 = null; @@ -757,7 +757,7 @@ public class BasicFunctionalityTest exte rsp = new SolrQueryResponse(); core.execute(core.getRequestHandler(req.getParams().get(CommonParams.QT)), req, rsp); - dl = ((ResultContext) rsp.getValues().get("response")).docs; + dl = ((ResultContext) rsp.getValues().get("response")).getDocList(); di = dl.iterator(); StoredDocument d2 = req.getSearcher().doc(di.nextDoc()); // ensure same doc, same lazy field now Modified: lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/TestDocumentBuilder.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/TestDocumentBuilder.java?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/TestDocumentBuilder.java (original) +++ lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/TestDocumentBuilder.java Wed Sep 2 13:06:13 2015 @@ -1,6 +1,6 @@ package org.apache.solr; -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. Modified: lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/TestGroupingSearch.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/TestGroupingSearch.java?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/TestGroupingSearch.java (original) +++ lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/TestGroupingSearch.java Wed Sep 2 13:06:13 2015 @@ -264,7 +264,7 @@ public class TestGroupingSearch extends SolrRequestInfo.clearRequestInfo(); } - assertEquals(6, ((ResultContext) response.getValues().get("response")).docs.matches()); + assertEquals(6, ((ResultContext) response.getValues().get("response")).getDocList().matches()); new BinaryResponseParser().processResponse(new ByteArrayInputStream(out.toByteArray()), ""); out.close(); } Modified: lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/TestJoin.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/TestJoin.java?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/TestJoin.java (original) +++ lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/TestJoin.java Wed Sep 2 13:06:13 2015 @@ -42,6 +42,7 @@ public class TestJoin extends SolrTestCa initCore("solrconfig.xml","schema12.xml"); } + @Test public void testJoin() throws Exception { assertU(add(doc("id", "1","name", "john", "title", "Director", "dept_s","Engineering"))); @@ -223,7 +224,7 @@ public class TestJoin extends SolrTestCa + "\n\trequest="+req + "\n\tresult="+strResponse + "\n\texpected="+ JSONUtil.toJSON(resultSet) - + "\n\tmodel="+ JSONUtil.toJSON(model) + + "\n\tmodel="+ model ); // re-execute the request... good for putting a breakpoint here for debugging Modified: lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/CdcrReplicationHandlerTest.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/CdcrReplicationHandlerTest.java?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/CdcrReplicationHandlerTest.java (original) +++ lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/CdcrReplicationHandlerTest.java Wed Sep 2 13:06:13 2015 @@ -1,4 +1,4 @@ -/** +/* * Copyright (c) 2015 Renaud Delbru. All Rights Reserved. */ package org.apache.solr.cloud; Modified: lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/DistribJoinFromCollectionTest.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/DistribJoinFromCollectionTest.java?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/DistribJoinFromCollectionTest.java (original) +++ lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/DistribJoinFromCollectionTest.java Wed Sep 2 13:06:13 2015 @@ -17,13 +17,7 @@ package org.apache.solr.cloud; * limitations under the License. */ -import org.apache.lucene.util.LuceneTestCase.Slow; -import org.apache.solr.JSONTestUtil; -import org.apache.solr.SolrTestCaseJ4.SuppressSSL; -import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.embedded.JettySolrRunner; -import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.QueryRequest; @@ -36,31 +30,16 @@ import org.apache.solr.common.SolrInputD import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.cloud.ZkCoreNodeProps; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.servlet.SolrDispatchFilter; import org.junit.After; import org.junit.Before; import org.apache.commons.lang.StringUtils; import org.junit.Test; +import static org.hamcrest.CoreMatchers.*; -import java.io.File; import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; import java.util.HashSet; -import java.util.List; -import java.util.Map; import java.util.Set; -import java.util.concurrent.TimeUnit; /** * Tests using fromIndex that points to a collection in SolrCloud mode. @@ -114,15 +93,46 @@ public class DistribJoinFromCollectionTe Thread.sleep(1000); // so the commits fire + //without score + testJoins(toColl, fromColl, toDocId, false); + + //with score + testJoins(toColl, fromColl, toDocId, true); + + log.info("DistribJoinFromCollectionTest logic complete ... deleting the " + toColl + " and " + fromColl + " collections"); + + // try to clean up + for (String c : new String[]{ toColl, fromColl }) { + try { + CollectionAdminRequest.Delete req = new CollectionAdminRequest.Delete() + .setCollectionName(c); + req.process(cloudClient); + } catch (Exception e) { + // don't fail the test + log.warn("Could not delete collection {} after test completed due to: " + e, c); + } + } + + log.info("DistribJoinFromCollectionTest succeeded ... shutting down now!"); + } + + private void testJoins(String toColl, String fromColl, Integer toDocId, boolean isScoresTest) + throws SolrServerException, IOException { // verify the join with fromIndex works - String joinQ = "{!join from=join_s fromIndex="+fromColl+" to=join_s}match_s:c"; - QueryRequest qr = new QueryRequest(params("collection", toColl, "q", joinQ, "fl", "id,get_s")); + final String[] scoreModes = {"avg","max","min","total"}; + String joinQ = "{!join " + anyScoreMode(isScoresTest, scoreModes) + + "from=join_s fromIndex=" + fromColl + " to=join_s}match_s:c"; + QueryRequest qr = new QueryRequest(params("collection", toColl, "q", joinQ, "fl", "id,get_s,score")); QueryResponse rsp = new QueryResponse(cloudClient.request(qr), cloudClient); SolrDocumentList hits = rsp.getResults(); assertTrue("Expected 1 doc", hits.getNumFound() == 1); SolrDocument doc = hits.get(0); assertEquals(toDocId, doc.getFirstValue("id")); assertEquals("b", doc.getFirstValue("get_s")); + assertScore(isScoresTest, doc); + + //negative test before creating an alias + checkAbsentFromIndex(fromColl, toColl, isScoresTest, scoreModes); // create an alias for the fromIndex and then query through the alias String alias = fromColl+"Alias"; @@ -131,37 +141,53 @@ public class DistribJoinFromCollectionTe request.setAliasedCollections(fromColl); request.process(cloudClient); - joinQ = "{!join from=join_s fromIndex="+alias+" to=join_s}match_s:c"; - qr = new QueryRequest(params("collection", toColl, "q", joinQ, "fl", "id,get_s")); + joinQ = "{!join " + anyScoreMode(isScoresTest, scoreModes) + + "from=join_s fromIndex=" + alias + " to=join_s}match_s:c"; + qr = new QueryRequest(params("collection", toColl, "q", joinQ, "fl", "id,get_s,score")); rsp = new QueryResponse(cloudClient.request(qr), cloudClient); hits = rsp.getResults(); assertTrue("Expected 1 doc", hits.getNumFound() == 1); doc = hits.get(0); assertEquals(toDocId, doc.getFirstValue("id")); assertEquals("b", doc.getFirstValue("get_s")); + assertScore(isScoresTest, doc); + + //negative test after creating an alias + checkAbsentFromIndex(fromColl, toColl, isScoresTest, scoreModes); // verify join doesn't work if no match in the "from" index - joinQ = "{!join from=join_s fromIndex="+fromColl+" to=join_s}match_s:d"; - qr = new QueryRequest(params("collection", toColl, "q", joinQ, "fl", "id,get_s")); + joinQ = "{!join " + (anyScoreMode(isScoresTest, scoreModes)) + + "from=join_s fromIndex=" + fromColl + " to=join_s}match_s:d"; + qr = new QueryRequest(params("collection", toColl, "q", joinQ, "fl", "id,get_s,score")); rsp = new QueryResponse(cloudClient.request(qr), cloudClient); hits = rsp.getResults(); assertTrue("Expected no hits", hits.getNumFound() == 0); + assertScore(isScoresTest, doc); + } - log.info("DistribJoinFromCollectionTest logic complete ... deleting the " + toColl + " and " + fromColl + " collections"); - - // try to clean up - for (String c : new String[]{ toColl, fromColl }) { - try { - CollectionAdminRequest.Delete req = new CollectionAdminRequest.Delete() - .setCollectionName(c); - req.process(cloudClient); - } catch (Exception e) { - // don't fail the test - log.warn("Could not delete collection {} after test completed due to: "+e, c); - } + private void assertScore(boolean isScoresTest, SolrDocument doc) { + if (isScoresTest) { + assertThat(doc.getFirstValue("score").toString(), not("1.0")); + } else { + assertEquals("1.0", doc.getFirstValue("score").toString()); } + } - log.info("DistribJoinFromCollectionTest succeeded ... shutting down now!"); + private String anyScoreMode(boolean isScoresTest, String[] scoreModes) { + return isScoresTest ? "score=" + (scoreModes[random().nextInt(scoreModes.length)]) + " " : ""; + } + + private void checkAbsentFromIndex(String fromColl, String toColl, boolean isScoresTest, String[] scoreModes) throws SolrServerException, IOException { + final String wrongName = fromColl + "WrongName"; + final String joinQ = "{!join " + (anyScoreMode(isScoresTest, scoreModes)) + + "from=join_s fromIndex=" + wrongName + " to=join_s}match_s:c"; + final QueryRequest qr = new QueryRequest(params("collection", toColl, "q", joinQ, "fl", "id,get_s,score")); + try { + cloudClient.request(qr); + } catch (HttpSolrClient.RemoteSolrException ex) { + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, ex.code()); + assertTrue(ex.getMessage().contains(wrongName)); + } } protected Integer indexDoc(String collection, int id, String joinField, String matchField, String getField) throws Exception { Modified: lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/DistributedQueueTest.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/DistributedQueueTest.java?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/DistributedQueueTest.java (original) +++ lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/DistributedQueueTest.java Wed Sep 2 13:06:13 2015 @@ -16,13 +16,18 @@ package org.apache.solr.cloud; * the License. */ -import java.io.File; import java.nio.charset.Charset; +import java.util.NoSuchElementException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.cloud.DistributedQueue.QueueEvent; - +import org.apache.solr.common.util.ExecutorUtil; +import org.apache.solr.common.util.SolrjNamedThreadFactory; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -33,6 +38,7 @@ public class DistributedQueueTest extend protected ZkTestServer zkServer; protected SolrZkClient zkClient; + protected ExecutorService executor = ExecutorUtil.newMDCAwareSingleThreadExecutor(new SolrjNamedThreadFactory("dqtest-")); @Before @Override @@ -44,37 +50,110 @@ public class DistributedQueueTest extend @Test public void testDistributedQueue() throws Exception { String dqZNode = "/distqueue/test"; - String testData = "hello world"; - long timeoutMs = 500L; + byte[] data = "hello world".getBytes(UTF8); - DistributedQueue dq = new DistributedQueue(zkClient, setupDistributedQueueZNode(dqZNode)); + DistributedQueue dq = makeDistributedQueue(dqZNode); // basic ops - assertTrue(dq.poll() == null); - byte[] data = testData.getBytes(UTF8); + assertNull(dq.poll()); + try { + dq.remove(); + fail("NoSuchElementException expected"); + } catch (NoSuchElementException expected) { + // expected + } + + dq.offer(data); + assertArrayEquals(dq.peek(500), data); + assertArrayEquals(dq.remove(), data); + assertNull(dq.poll()); + dq.offer(data); - assertEquals(new String(dq.peek(),UTF8), testData); - assertEquals(new String(dq.take(),UTF8), testData); - assertTrue(dq.poll() == null); - QueueEvent qe = dq.offer(data, timeoutMs); - assertNotNull(qe); - assertEquals(new String(dq.remove(),UTF8), testData); + assertArrayEquals(dq.take(), data); // waits for data + assertNull(dq.poll()); + + dq.offer(data); + dq.peek(true); // wait until data is definitely there before calling remove + assertArrayEquals(dq.remove(), data); + assertNull(dq.poll()); // should block until the background thread makes the offer (new QueueChangerThread(dq, 1000)).start(); - qe = dq.peek(true); - assertNotNull(qe); - dq.remove(); + assertNotNull(dq.peek(true)); + assertNotNull(dq.remove()); + assertNull(dq.poll()); // timeout scenario ... background thread won't offer until long after the peek times out QueueChangerThread qct = new QueueChangerThread(dq, 1000); qct.start(); - qe = dq.peek(500); - assertTrue(qe == null); + assertNull(dq.peek(500)); + qct.join(); + } + + @Test + public void testDistributedQueueBlocking() throws Exception { + String dqZNode = "/distqueue/test"; + String testData = "hello world"; + DistributedQueue dq = makeDistributedQueue(dqZNode); + + assertNull(dq.peek()); + Future future = executor.submit(() -> new String(dq.peek(true), UTF8)); try { - qct.interrupt(); - } catch (Exception exc) {} + future.get(1000, TimeUnit.MILLISECONDS); + fail("TimeoutException expected"); + } catch (TimeoutException expected) { + assertFalse(future.isDone()); + } + + // Ultimately trips the watcher, triggering child refresh + dq.offer(testData.getBytes(UTF8)); + assertEquals(testData, future.get(1000, TimeUnit.MILLISECONDS)); + assertNotNull(dq.poll()); + + // After draining the queue, a watcher should be set. + assertNull(dq.peek(100)); + assertTrue(dq.hasWatcher()); + + forceSessionExpire(); + + // Session expiry should have fired the watcher. + Thread.sleep(100); + assertFalse(dq.hasWatcher()); + + // Rerun the earlier test make sure updates are still seen, post reconnection. + future = executor.submit(() -> new String(dq.peek(true), UTF8)); + try { + future.get(1000, TimeUnit.MILLISECONDS); + fail("TimeoutException expected"); + } catch (TimeoutException expected) { + assertFalse(future.isDone()); + } + + // Ultimately trips the watcher, triggering child refresh + dq.offer(testData.getBytes(UTF8)); + assertEquals(testData, future.get(1000, TimeUnit.MILLISECONDS)); + assertNotNull(dq.poll()); + assertNull(dq.poll()); + } + + private void forceSessionExpire() throws InterruptedException, TimeoutException { + long sessionId = zkClient.getSolrZooKeeper().getSessionId(); + zkServer.expire(sessionId); + zkClient.getConnectionManager().waitForDisconnected(10000); + zkClient.getConnectionManager().waitForConnected(10000); + for (int i = 0; i < 100; ++i) { + if (zkClient.isConnected()) { + break; + } + Thread.sleep(50); + } + assertTrue(zkClient.isConnected()); + assertFalse(sessionId == zkClient.getSolrZooKeeper().getSessionId()); + } + + protected DistributedQueue makeDistributedQueue(String dqZNode) throws Exception { + return new DistributedQueue(zkClient, setupNewDistributedQueueZNode(dqZNode)); } private class QueueChangerThread extends Thread { @@ -99,7 +178,7 @@ public class DistributedQueueTest extend } } - protected String setupDistributedQueueZNode(String znodePath) throws Exception { + protected String setupNewDistributedQueueZNode(String znodePath) throws Exception { if (!zkClient.exists("/", true)) zkClient.makePath("/", false, true); if (zkClient.exists(znodePath, true)) @@ -113,8 +192,10 @@ public class DistributedQueueTest extend public void tearDown() throws Exception { try { super.tearDown(); - } catch (Exception exc) {} + } catch (Exception exc) { + } closeZk(); + executor.shutdown(); } protected void setupZk() throws Exception { Modified: lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java (original) +++ lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java Wed Sep 2 13:06:13 2015 @@ -43,8 +43,8 @@ import java.util.Locale; import java.util.Map; import java.util.concurrent.TimeUnit; -import static org.apache.solr.cloud.OverseerCollectionProcessor.getLeaderNode; -import static org.apache.solr.cloud.OverseerCollectionProcessor.getSortedOverseerNodeNames; +import static org.apache.solr.cloud.OverseerCollectionConfigSetProcessor.getLeaderNode; +import static org.apache.solr.cloud.OverseerCollectionConfigSetProcessor.getSortedOverseerNodeNames; import static org.apache.solr.cloud.OverseerCollectionMessageHandler.NUM_SLICES; import static org.apache.solr.common.util.Utils.makeMap; import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE; @@ -88,7 +88,7 @@ public class OverseerRolesTest extends final TimeOut timeout = new TimeOut(10, TimeUnit.SECONDS); String newLeader=null; for(;! timeout.hasTimedOut();){ - newLeader = OverseerCollectionProcessor.getLeaderNode(zk); + newLeader = OverseerCollectionConfigSetProcessor.getLeaderNode(zk); if(newLeader!=null && !newLeader.equals(leader)) break; Thread.sleep(100); } @@ -96,7 +96,7 @@ public class OverseerRolesTest extends - assertTrue("The old leader should have rejoined election ", OverseerCollectionProcessor.getSortedOverseerNodeNames(zk).contains(leader)); + assertTrue("The old leader should have rejoined election ", OverseerCollectionConfigSetProcessor.getSortedOverseerNodeNames(zk).contains(leader)); } @@ -108,10 +108,10 @@ public class OverseerRolesTest extends createCollection(collectionName, client); waitForRecoveriesToFinish(collectionName, false); - List l = OverseerCollectionProcessor.getSortedOverseerNodeNames(client.getZkStateReader().getZkClient()) ; + List l = OverseerCollectionConfigSetProcessor.getSortedOverseerNodeNames(client.getZkStateReader().getZkClient()) ; log.info("All nodes {}", l); - String currentLeader = OverseerCollectionProcessor.getLeaderNode(client.getZkStateReader().getZkClient()); + String currentLeader = OverseerCollectionConfigSetProcessor.getLeaderNode(client.getZkStateReader().getZkClient()); log.info("Current leader {} ", currentLeader); l.remove(currentLeader); @@ -124,7 +124,7 @@ public class OverseerRolesTest extends boolean leaderchanged = false; for(;!timeout.hasTimedOut();){ - if(overseerDesignate.equals(OverseerCollectionProcessor.getLeaderNode(client.getZkStateReader().getZkClient()))){ + if(overseerDesignate.equals(OverseerCollectionConfigSetProcessor.getLeaderNode(client.getZkStateReader().getZkClient()))){ log.info("overseer designate is the new overseer"); leaderchanged =true; break; @@ -134,7 +134,7 @@ public class OverseerRolesTest extends assertTrue("could not set the new overseer . expected "+ overseerDesignate + " current order : " + getSortedOverseerNodeNames(client.getZkStateReader().getZkClient()) + - " ldr :"+ OverseerCollectionProcessor.getLeaderNode(client.getZkStateReader().getZkClient()) ,leaderchanged); + " ldr :"+ OverseerCollectionConfigSetProcessor.getLeaderNode(client.getZkStateReader().getZkClient()) ,leaderchanged); @@ -176,7 +176,7 @@ public class OverseerRolesTest extends log.info("leader node {}", leaderJetty.getBaseUrl()); log.info ("current election Queue", - OverseerCollectionProcessor.getSortedElectionNodes(client.getZkStateReader().getZkClient(), + OverseerCollectionConfigSetProcessor.getSortedElectionNodes(client.getZkStateReader().getZkClient(), OverseerElectionContext.PATH + LeaderElector.ELECTION_NODE)); ChaosMonkey.stop(leaderJetty); timeout = new TimeOut(10, TimeUnit.SECONDS); Modified: lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java (original) +++ lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java Wed Sep 2 13:06:13 2015 @@ -17,6 +17,7 @@ package org.apache.solr.cloud; * limitations under the License. */ +import javax.xml.parsers.ParserConfigurationException; import java.io.File; import java.io.IOException; import java.util.ArrayList; @@ -31,8 +32,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; -import javax.xml.parsers.ParserConfigurationException; - import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.cloud.overseer.OverseerAction; @@ -58,6 +57,7 @@ import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.KeeperException.NoNodeException; import org.apache.zookeeper.KeeperException.NodeExistsException; +import org.apache.zookeeper.data.Stat; import org.junit.After; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -573,7 +573,7 @@ public class OverseerTest extends SolrTe q.offer(Utils.toJSON(m)); - verifyStatus(reader, Replica.State.ACTIVE); + verifyStatus(reader, "collection1", "shard1", "core_node1", Replica.State.ACTIVE); } finally { @@ -585,20 +585,20 @@ public class OverseerTest extends SolrTe } } - private void verifyStatus(ZkStateReader reader, Replica.State expectedState) throws InterruptedException { + private void verifyStatus(ZkStateReader reader, String collection, String shard, String coreNodeName, Replica.State expectedState) throws InterruptedException { int maxIterations = 100; Replica.State coreState = null; while(maxIterations-->0) { - Slice slice = reader.getClusterState().getSlice("collection1", "shard1"); + Slice slice = reader.getClusterState().getSlice(collection, shard); if(slice!=null) { - coreState = slice.getReplicasMap().get("core_node1").getState(); + coreState = slice.getReplicasMap().get(coreNodeName).getState(); if(coreState == expectedState) { return; } } Thread.sleep(50); } - fail("Illegal state, was:" + coreState + " expected:" + expectedState + "clusterState:" + reader.getClusterState()); + fail("Illegal state, was: " + coreState + " expected:" + expectedState + " clusterState:" + reader.getClusterState()); } private void verifyShardLeader(ZkStateReader reader, String collection, String shard, String expectedCore) throws InterruptedException, KeeperException { @@ -649,7 +649,7 @@ public class OverseerTest extends SolrTe mockController.publishState(collection, "core1", "core_node1", Replica.State.RECOVERING, 1); waitForCollections(reader, collection); - verifyStatus(reader, Replica.State.RECOVERING); + verifyStatus(reader, collection, "shard1", "core_node1", Replica.State.RECOVERING); int version = getClusterStateVersion(zkClient); @@ -657,7 +657,7 @@ public class OverseerTest extends SolrTe while (version == getClusterStateVersion(zkClient)); - verifyStatus(reader, Replica.State.ACTIVE); + verifyStatus(reader, collection, "shard1", "core_node1", Replica.State.ACTIVE); version = getClusterStateVersion(zkClient); overseerClient.close(); Thread.sleep(1000); // wait for overseer to get killed @@ -669,7 +669,7 @@ public class OverseerTest extends SolrTe while (version == getClusterStateVersion(zkClient)); - verifyStatus(reader, Replica.State.RECOVERING); + verifyStatus(reader, collection, "shard1", "core_node1", Replica.State.RECOVERING); assertEquals("Live nodes count does not match", 1, reader .getClusterState().getLiveNodes().size()); @@ -680,7 +680,7 @@ public class OverseerTest extends SolrTe while (version == getClusterStateVersion(zkClient)); Thread.sleep(500); assertFalse("collection1 should be gone after publishing the null state", - reader.getClusterState().getCollections().contains(collection)); + reader.getClusterState().hasCollection(collection)); } finally { close(mockController); close(overseerClient); @@ -884,7 +884,7 @@ public class OverseerTest extends SolrTe waitForCollections(reader, "collection1"); - verifyStatus(reader, Replica.State.RECOVERING); + verifyStatus(reader, collection, "shard1", "core_node1", Replica.State.RECOVERING); mockController.close(); @@ -1161,6 +1161,111 @@ public class OverseerTest extends SolrTe close(reader); server.shutdown(); } + } + + @Test + public void testExternalClusterStateChangeBehavior() throws Exception { + String zkDir = createTempDir("testExternalClusterStateChangeBehavior").toFile().getAbsolutePath(); + + ZkTestServer server = new ZkTestServer(zkDir); + + SolrZkClient zkClient = null; + ZkStateReader reader = null; + SolrZkClient overseerClient = null; + + try { + server.run(); + zkClient = new SolrZkClient(server.getZkAddress(), TIMEOUT); + + AbstractZkTestCase.tryCleanSolrZkNode(server.getZkHost()); + AbstractZkTestCase.makeSolrZkNode(server.getZkHost()); + ZkController.createClusterZkNodes(zkClient); + + zkClient.create("/collections/test", null, CreateMode.PERSISTENT, true); + + reader = new ZkStateReader(zkClient); + reader.createClusterStateWatchersAndUpdate(); + + overseerClient = electNewOverseer(server.getZkAddress()); + + DistributedQueue q = Overseer.getInQueue(zkClient); + + ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, OverseerAction.STATE.toLower(), + ZkStateReader.BASE_URL_PROP, "http://127.0.0.1/solr", + ZkStateReader.NODE_NAME_PROP, "node1", + ZkStateReader.COLLECTION_PROP, "c1", + ZkStateReader.CORE_NAME_PROP, "core1", + ZkStateReader.ROLES_PROP, "", + ZkStateReader.STATE_PROP, Replica.State.DOWN.toString()); + + q.offer(Utils.toJSON(m)); + + waitForCollections(reader, "c1"); + verifyStatus(reader, "c1", "shard1", "core_node1", Replica.State.DOWN); + + m = new ZkNodeProps(Overseer.QUEUE_OPERATION, OverseerAction.STATE.toLower(), + ZkStateReader.BASE_URL_PROP, "http://127.0.0.1/solr", + ZkStateReader.NODE_NAME_PROP, "node1", + ZkStateReader.COLLECTION_PROP, "c1", + ZkStateReader.CORE_NAME_PROP, "core1", + ZkStateReader.ROLES_PROP, "", + ZkStateReader.STATE_PROP, Replica.State.RECOVERING.toString()); + + q.offer(Utils.toJSON(m)); + + + m = new ZkNodeProps(Overseer.QUEUE_OPERATION, OverseerAction.STATE.toLower(), + ZkStateReader.BASE_URL_PROP, "http://127.0.0.1/solr", + ZkStateReader.NODE_NAME_PROP, "node1", + ZkStateReader.COLLECTION_PROP, "c1", + ZkStateReader.CORE_NAME_PROP, "core1", + ZkStateReader.ROLES_PROP, "", + ZkStateReader.STATE_PROP, Replica.State.ACTIVE.toString()); + + q.offer(Utils.toJSON(m)); + + Stat stat = new Stat(); + byte[] data = zkClient.getData("/clusterstate.json", null, stat, true); + // Simulate an external modification + zkClient.setData("/clusterstate.json", data, true); + + m = new ZkNodeProps(Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.CREATE.toLower(), + "name", "test", + ZkStateReader.NUM_SHARDS_PROP, "1", + ZkStateReader.REPLICATION_FACTOR, "1", + DocCollection.STATE_FORMAT, "2" + ); + q.offer(Utils.toJSON(m)); + + m = new ZkNodeProps(Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.CREATESHARD.toLower(), + "collection", "test", + ZkStateReader.SHARD_ID_PROP, "x", + ZkStateReader.REPLICATION_FACTOR, "1" + ); + q.offer(Utils.toJSON(m)); + + m = new ZkNodeProps(Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.ADDREPLICA.toLower(), + "collection", "test", + ZkStateReader.SHARD_ID_PROP, "x", + ZkStateReader.BASE_URL_PROP, "http://127.0.0.1/solr", + ZkStateReader.NODE_NAME_PROP, "node1", + ZkStateReader.CORE_NAME_PROP, "core1", + ZkStateReader.STATE_PROP, Replica.State.DOWN.toString() + ); + q.offer(Utils.toJSON(m)); + + waitForCollections(reader, "test"); + verifyStatus(reader, "test", "x", "core_node1", Replica.State.DOWN); + + waitForCollections(reader, "c1"); + verifyStatus(reader, "c1", "shard1", "core_node1", Replica.State.ACTIVE); + + } finally { + close(zkClient); + close(overseerClient); + close(reader); + server.shutdown(); + } } private void close(ZkStateReader reader) { Modified: lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/RollingRestartTest.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/RollingRestartTest.java?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/RollingRestartTest.java (original) +++ lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/RollingRestartTest.java Wed Sep 2 13:06:13 2015 @@ -57,7 +57,7 @@ public class RollingRestartTest extends public void restartWithRolesTest() throws Exception { - String leader = OverseerCollectionProcessor.getLeaderNode(cloudClient.getZkStateReader().getZkClient()); + String leader = OverseerCollectionConfigSetProcessor.getLeaderNode(cloudClient.getZkStateReader().getZkClient()); assertNotNull(leader); log.info("Current overseer leader = {}", leader); @@ -93,10 +93,10 @@ public class RollingRestartTest extends sawLiveDesignate = true; boolean success = waitUntilOverseerDesignateIsLeader(cloudClient.getZkStateReader().getZkClient(), designates, MAX_WAIT_TIME); if (!success) { - leader = OverseerCollectionProcessor.getLeaderNode(cloudClient.getZkStateReader().getZkClient()); + leader = OverseerCollectionConfigSetProcessor.getLeaderNode(cloudClient.getZkStateReader().getZkClient()); if (leader == null) log.error("NOOVERSEER election queue is :" + - OverseerCollectionProcessor.getSortedElectionNodes(cloudClient.getZkStateReader().getZkClient(), + OverseerCollectionConfigSetProcessor.getSortedElectionNodes(cloudClient.getZkStateReader().getZkClient(), OverseerElectionContext.PATH + LeaderElector.ELECTION_NODE)); fail("No overseer designate as leader found after restart #" + (i + 1) + ": " + leader); } @@ -104,10 +104,10 @@ public class RollingRestartTest extends assertTrue("Unable to restart (#" + i + "): " + cloudJetty, ChaosMonkey.start(cloudJetty.jetty)); boolean success = waitUntilOverseerDesignateIsLeader(cloudClient.getZkStateReader().getZkClient(), designates, MAX_WAIT_TIME); if (!success) { - leader = OverseerCollectionProcessor.getLeaderNode(cloudClient.getZkStateReader().getZkClient()); + leader = OverseerCollectionConfigSetProcessor.getLeaderNode(cloudClient.getZkStateReader().getZkClient()); if (leader == null) log.error("NOOVERSEER election queue is :" + - OverseerCollectionProcessor.getSortedElectionNodes(cloudClient.getZkStateReader().getZkClient(), + OverseerCollectionConfigSetProcessor.getSortedElectionNodes(cloudClient.getZkStateReader().getZkClient(), OverseerElectionContext.PATH + LeaderElector.ELECTION_NODE)); fail("No overseer leader found after restart #" + (i + 1) + ": " + leader); } @@ -120,7 +120,7 @@ public class RollingRestartTest extends assertTrue("Test may not be working if we never saw a live designate", sawLiveDesignate); - leader = OverseerCollectionProcessor.getLeaderNode(cloudClient.getZkStateReader().getZkClient()); + leader = OverseerCollectionConfigSetProcessor.getLeaderNode(cloudClient.getZkStateReader().getZkClient()); assertNotNull(leader); log.info("Current overseer leader (after restart) = {}", leader); @@ -135,7 +135,7 @@ public class RollingRestartTest extends int stableCheckTimeout = 2000; String oldleader = null; while (System.nanoTime() < timeout && System.nanoTime() < maxTimeout) { - String newLeader = OverseerCollectionProcessor.getLeaderNode(testZkClient); + String newLeader = OverseerCollectionConfigSetProcessor.getLeaderNode(testZkClient); if (newLeader != null && !newLeader.equals(oldleader)) { // the leaders have changed, let's move the timeout further timeout = System.nanoTime() + TimeUnit.NANOSECONDS.convert(60, TimeUnit.SECONDS); Modified: lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/SimpleCollectionCreateDeleteTest.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/SimpleCollectionCreateDeleteTest.java?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/SimpleCollectionCreateDeleteTest.java (original) +++ lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/SimpleCollectionCreateDeleteTest.java Wed Sep 2 13:06:13 2015 @@ -33,7 +33,7 @@ public class SimpleCollectionCreateDelet @Test @ShardsFixed(num = 1) public void test() throws Exception { - String overseerNode = OverseerCollectionProcessor.getLeaderNode(cloudClient.getZkStateReader().getZkClient()); + String overseerNode = OverseerCollectionConfigSetProcessor.getLeaderNode(cloudClient.getZkStateReader().getZkClient()); String notOverseerNode = null; for (CloudJettyRunner cloudJetty : cloudJettys) { if (!overseerNode.equals(cloudJetty.nodeName)) { Modified: lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/TestCloudPivotFacet.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/TestCloudPivotFacet.java?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/TestCloudPivotFacet.java (original) +++ lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/TestCloudPivotFacet.java Wed Sep 2 13:06:13 2015 @@ -291,7 +291,7 @@ public class TestCloudPivotFacet extends } /** - * Recursive Helper method for asserting that pivot constraint counds match + * Recursive Helper method for asserting that pivot constraint counts match * results when filtering on those constraints. Returns the recursive depth reached * (for sanity checking) */ Modified: lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/TestLeaderElectionZkExpiry.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/TestLeaderElectionZkExpiry.java?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/TestLeaderElectionZkExpiry.java (original) +++ lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/TestLeaderElectionZkExpiry.java Wed Sep 2 13:06:13 2015 @@ -78,7 +78,7 @@ public class TestLeaderElectionZkExpiry boolean found = false; while (System.nanoTime() < timeout) { try { - String leaderNode = OverseerCollectionProcessor.getLeaderNode(zc); + String leaderNode = OverseerCollectionConfigSetProcessor.getLeaderNode(zc); if (leaderNode != null && !leaderNode.trim().isEmpty()) { log.info("Time={} Overseer leader is = {}", System.nanoTime(), leaderNode); found = true; Modified: lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/TestRebalanceLeaders.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/TestRebalanceLeaders.java?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/TestRebalanceLeaders.java (original) +++ lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/TestRebalanceLeaders.java Wed Sep 2 13:06:13 2015 @@ -16,6 +16,13 @@ package org.apache.solr.cloud; * limitations under the License. */ +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudSolrClient; @@ -29,17 +36,12 @@ import org.apache.solr.common.util.Utils import org.apache.solr.util.TimeOut; import org.apache.zookeeper.KeeperException; import org.junit.Test; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class TestRebalanceLeaders extends AbstractFullDistribZkTestBase { - + private static Logger log = LoggerFactory.getLogger(TestRebalanceLeaders.class); public static final String COLLECTION_NAME = "testcollection"; public TestRebalanceLeaders() { @@ -71,7 +73,6 @@ public class TestRebalanceLeaders extend waitForRecoveriesToFinish(COLLECTION_NAME, false); listCollection(); - rebalanceLeaderTest(); } @@ -117,16 +118,21 @@ public class TestRebalanceLeaders extend // 3> The node that ZooKeeper thinks is the leader is the one we think should be the leader. void checkConsistency() throws InterruptedException, KeeperException { TimeOut timeout = new TimeOut(timeoutMs, TimeUnit.MILLISECONDS); - - while (! timeout.hasTimedOut()) { - if (checkAppearOnce() && - checkElectionZero() && - checkZkLeadersAgree()) { + boolean checkAppearOnce = false; + boolean checkElectionZero = false; + boolean checkZkLeadersAgree = false; + while (!timeout.hasTimedOut()) { + checkAppearOnce = checkAppearOnce(); + checkElectionZero = checkElectionZero(); + checkZkLeadersAgree = checkZkLeadersAgree(); + if (checkAppearOnce && checkElectionZero && checkZkLeadersAgree) { return; } Thread.sleep(1000); } - fail("Checking the rebalance leader command failed"); + + fail("Checking the rebalance leader command failed, checkAppearOnce=" + checkAppearOnce + " checkElectionZero=" + + checkElectionZero + " checkZkLeadersAgree=" + checkZkLeadersAgree); } @@ -182,7 +188,7 @@ public class TestRebalanceLeaders extend List getOverseerSort(String key) { List ret = null; try { - ret = OverseerCollectionProcessor.getSortedElectionNodes(cloudClient.getZkStateReader().getZkClient(), + ret = OverseerCollectionConfigSetProcessor.getSortedElectionNodes(cloudClient.getZkStateReader().getZkClient(), "/collections/" + COLLECTION_NAME + "/leader_elect/" + key + "/election"); return ret; } catch (KeeperException e) { @@ -211,25 +217,26 @@ public class TestRebalanceLeaders extend // Do who we _think_ should be the leader agree with the leader nodes? Boolean checkZkLeadersAgree() throws KeeperException, InterruptedException { - for (Map.Entry ent : expected.entrySet()) { - - String path = "/collections/" + COLLECTION_NAME + "/leaders/" + ent.getKey(); + for (Map.Entry ent : expected.entrySet()) { + + String path = "/collections/" + COLLECTION_NAME + "/leaders/" + ent.getKey() + "/leader"; byte[] data = getZkData(cloudClient, path); - if (data == null) return false; - + if (data == null) { + log.warn("path to check not found {}", path); + return false; + } + String repCore = null; String zkCore = null; - - if (data == null) { + + Map m = (Map) Utils.fromJSON(data); + zkCore = (String) m.get("core"); + repCore = ent.getValue().getStr("core"); + if (zkCore.equals(repCore) == false) { + log.warn("leader in zk does not match what we expect: {} != {}", zkCore, repCore); return false; - } else { - Map m = (Map) Utils.fromJSON(data); - zkCore = (String) m.get("core"); - repCore = ent.getValue().getStr("core"); - if (zkCore.equals(repCore) == false) { - return false; - } } + } return true; } Modified: lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/overseer/ZkStateReaderTest.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/overseer/ZkStateReaderTest.java?rev=1700800&r1=1700799&r2=1700800&view=diff ============================================================================== --- lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/overseer/ZkStateReaderTest.java (original) +++ lucene/dev/branches/lucene6699/solr/core/src/test/org/apache/solr/cloud/overseer/ZkStateReaderTest.java Wed Sep 2 13:06:13 2015 @@ -39,15 +39,25 @@ public class ZkStateReaderTest extends S /** Uses explicit refresh to ensure latest changes are visible. */ public void testStateFormatUpdateWithExplicitRefresh() throws Exception { - testStateFormatUpdate(true); + testStateFormatUpdate(true, true); + } + + /** Uses explicit refresh to ensure latest changes are visible. */ + public void testStateFormatUpdateWithExplicitRefreshLazy() throws Exception { + testStateFormatUpdate(true, false); } /** ZkStateReader should automatically pick up changes based on ZK watches. */ public void testStateFormatUpdateWithTimeDelay() throws Exception { - testStateFormatUpdate(false); + testStateFormatUpdate(false, true); } - public void testStateFormatUpdate(boolean explicitRefresh) throws Exception { + /** ZkStateReader should automatically pick up changes based on ZK watches. */ + public void testStateFormatUpdateWithTimeDelayLazy() throws Exception { + testStateFormatUpdate(false, false); + } + + public void testStateFormatUpdate(boolean explicitRefresh, boolean isInteresting) throws Exception { String zkDir = createTempDir("testStateFormatUpdate").toFile().getAbsolutePath(); ZkTestServer server = new ZkTestServer(zkDir); @@ -64,7 +74,9 @@ public class ZkStateReaderTest extends S ZkStateReader reader = new ZkStateReader(zkClient); reader.createClusterStateWatchersAndUpdate(); - int trackedStateVersion = reader.getClusterState().getZkClusterStateVersion(); + if (isInteresting) { + reader.addCollectionWatch("c1"); + } ZkStateWriter writer = new ZkStateWriter(reader, new Overseer.Stats()); @@ -82,7 +94,16 @@ public class ZkStateReaderTest extends S boolean exists = zkClient.exists(ZkStateReader.COLLECTIONS_ZKNODE + "/c1/state.json", true); assertFalse(exists); - trackedStateVersion = refresh(reader, trackedStateVersion, explicitRefresh); + if (explicitRefresh) { + reader.updateClusterState(); + } else { + for (int i = 0; i < 100; ++i) { + if (reader.getClusterState().hasCollection("c1")) { + break; + } + Thread.sleep(50); + } + } DocCollection collection = reader.getClusterState().getCollection("c1"); assertEquals(1, collection.getStateFormat()); @@ -101,7 +122,16 @@ public class ZkStateReaderTest extends S boolean exists = zkClient.exists(ZkStateReader.COLLECTIONS_ZKNODE + "/c1/state.json", true); assertTrue(exists); - trackedStateVersion = refresh(reader, trackedStateVersion, explicitRefresh); + if (explicitRefresh) { + reader.updateClusterState(); + } else { + for (int i = 0; i < 100; ++i) { + if (reader.getClusterState().getCollection("c1").getStateFormat() == 2) { + break; + } + Thread.sleep(50); + } + } DocCollection collection = reader.getClusterState().getCollection("c1"); assertEquals(2, collection.getStateFormat()); @@ -138,7 +168,7 @@ public class ZkStateReaderTest extends S new DocCollection("c1", new HashMap(), new HashMap(), DocRouter.DEFAULT, 0, ZkStateReader.COLLECTIONS_ZKNODE + "/c1/state.json")); writer.enqueueUpdate(reader.getClusterState(), c1, null); writer.writePendingUpdates(); - refresh(reader, 0, true); + reader.updateClusterState(); assertTrue(reader.getClusterState().getCollectionRef("c1").isLazilyLoaded()); reader.addCollectionWatch("c1"); @@ -151,20 +181,4 @@ public class ZkStateReaderTest extends S server.shutdown(); } } - - private static int refresh(ZkStateReader reader, int trackedStateVersion, boolean explicitRefresh) throws KeeperException, InterruptedException { - if (explicitRefresh) { - reader.updateClusterState(); - return reader.getClusterState().getZkClusterStateVersion(); - } - for (int i = 0; i < 100; ++i) { - // Loop until we observe the change. - int newStateVersion = reader.getClusterState().getZkClusterStateVersion(); - if (newStateVersion > trackedStateVersion) { - return newStateVersion; - } - Thread.sleep(100); - } - throw new AssertionError("Did not observe expected update"); - } }